From ab3658e17f78249ef30f4d6c11589b601f33e73c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Fri, 23 Oct 2026 18:27:14 +0300 Subject: [PATCH 01/25] docs: record reviewed source transports implementation contract --- .../source-transports-e2e/sources-plan.md | 207 ++++++++++++++++++ .../sources-review-r1.md | 130 +++++++++++ .../sources-review-r2.md | 85 +++++++ 3 files changed, 422 insertions(+) create mode 100644 docs/plans/source-transports-e2e/sources-plan.md create mode 100644 docs/plans/source-transports-e2e/sources-review-r1.md create mode 100644 docs/plans/source-transports-e2e/sources-review-r2.md diff --git a/docs/plans/source-transports-e2e/sources-plan.md b/docs/plans/source-transports-e2e/sources-plan.md new file mode 100644 index 000000000..ba9013d49 --- /dev/null +++ b/docs/plans/source-transports-e2e/sources-plan.md @@ -0,0 +1,207 @@ +# Sources PR research and implementation plan + +Status: both independent plan review/fix rounds are verified GO; implementation authorized by the owner on 2026-10-23. Publication remains separately coordinated. + +Repository: `777genius/universal-agent-plugins`; implementation base `3ea24e3014ddf6679ca86729c56f1069112b0d3a`; assigned worktree `the dedicated source implementation worktree`, branch `feat/source-transports-e2e`. + +## 1. Accepted scope and constraints + +Support complete standard Agent Plugins packages acquired through arbitrary public HTTPS Git, GitHub/GitLab/Azure browser repository/tree URLs, native SSH/private Git using the user's existing credentials, HTTPS package archives including extensionless downloads, and local ZIP/TAR/TAR.GZ/TGZ archives. This includes saved add, dependency intent and lock, transitive resolution, update, locked install/sync, export/import/fresh-store restore, dry run, repair and remove. + +No raw SKILL.md importer, remote-directory traversal protocol, YAML fallback, new plugin manifest, package execution, scanner uploads or edits to frozen VS Code implementation/configuration/protocols/dedicated fixtures. Standard `plugin.json` and component interpretation stay unchanged. HTTPS downloads contain a complete package or a repository-like wrapper around one; HTML or a standalone manifest is not a package. + +Directory and Discovery remain separately signed contracts. Their GitHub owner/repository + immutable SHA records, eligibility policy, anonymous acquisition, signature/digest checks and source keys do not broaden merely because direct user sources broaden. Direct user sources gain no Directory endorsement. + +Read required: root `AGENTS.md`, `docs/AUTHORING_CAPABILITY_PRESERVATION.md`, owner decisions at top of `docs/STANDARD_FIRST_AUTHORING_ENGINE_IMPLEMENTATION_PLAN.md`, existing `docs/UAP_DEPENDENCIES.md`, `docs/THREAT_MODEL.md`. User-selected GPT-6.1 high/xhigh supersedes historical worker-model text. + +## 2. Evidence from current source + +* `cli/internal/agentpluginscli/source.go` accepts explicit local paths, an exact GitHub `owner/repo@FULL_SHA[//path]`, Discovery and Directory. Root package discovery is bounded to 16 candidates and validates candidates with the real standard loader. +* `cli/internal/agentpluginscli/app.go:SourceAcquirer` exposes only local and GitHub operations. Existing fake sources implement this narrow interface throughout tests. +* `install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go` already separates a private `acquireGit(url, repository, revision, subpath)` implementation from its GitHub wrapper. It fetches an exact SHA, validates `FETCH_HEAD^{commit}`, tracks executable modes, seals snapshots, and never runs package commands. The implementation currently constructs GitHub-only canonical identity and failures. +* `sourceacquisition/refs.go` already resolves default branches, exact branches/tags, ambiguity and annotated tags. Adapt this neutral mechanism rather than inventing another resolver. +* Its `OSRunner` intentionally strips HOME, user Git config, SSH agent/askpass and all GIT_* variables. This is correct for the signed anonymous path and currently prevents private Git. Its contained process runner preserves cancellation/process-tree cleanup. +* Preserved `install/integrationctl/adapters/source/resolver_parse.go` supports `https://...git#ref`, `ssh://...git#ref` and `git@host:path.git#ref`; `resolver_clone.go` uses ambient clone/checkouts and error output. Reuse/adapt grammar ideas only. Do not route the standard engine through the old YAML resolver, copy its ambient clone flow or delete its code/tests. +* `environment/model.go` hardcodes `GitSource.Repository` and `LockedGitSource.Repository` to GitHub owner/repo, `github::` source keys, two source variants and commit/ref/default selectors. `environment/graph.go`, `resolve_graph.go`, `resolve_expand.go` and coordinator logic assume every non-Git source is Registry. +* `adapters/dependencymanifest/decode.go` turns a `git` string into GitHub owner/repo by trimming a fixed prefix/suffix. `cli/internal/agentpluginscli/environment_session.go` writes GitHub URLs through `manifestedit`. +* `deps_resolver.go`, `deps_staging.go`, `deps_sync.go` and `saved_add.go` reconstruct GitHub identities and fetch from GitHub even during fresh-store restore. These are necessary E2E boundaries. +* `adapters/environmentstore/store.go` is content addressed, seals and rechecks bytes and executable inventory, publishes atomically and does not need another cache layout. Source metadata is provenance, not its content key. +* `domain.SourceIdentity` and `SourceBinding` lack a typed source URL/archive checksum. Multiple explicit copy sites in `usecase/{installation,binding,group_commit}.go` must retain new source facts. +* `directoryv1.ResolveDirectExact` is a strict GitHub/local validator inside the signed adapter; new direct source validation should not weaken that existing contract. +* `cli/internal/app/bundle_install_archive_header.go` already contains robust retained TAR path/type/collision/size policy; `bundle_install_apply_extract*.go` contains private extraction. It expects a legacy export marker and gzip TAR. It cannot be imported by the lower-level installer module without reversing dependency direction. Preserve it and adapt the proven policy into a narrow source-acquisition archive adapter; do not create a legacy command dependency or require `.plugin-kit-ai-export.json`. + +## 3. Minimal design decisions for review + +### 3.1 One pure normalized source grammar + +Add neutral package `install/integrationctl/agentplugins/sourcespec` with no network, filesystem, domain or environment imports. Its immutable values normalize Git endpoints, package paths, selectors, archive locations and safe persistent display forms. It is shared by CLI, environment validation and acquisition; source parsing is not duplicated in three layers. + +Proposed value types: + +```go +type Kind string // git, archive, https-auto, local-archive +type Git struct { URL, Repository, Path, Ref, Commit string; Browser *BrowserTree } +type BrowserTree struct { Provider, Suffix string } // transient unresolved selector/path +type Archive struct { URL, Path, SHA256 string } // URL is HTTPS or explicit absolute local file +type Spec struct { Kind Kind; Git *Git; Archive *Archive; Requested string } +func Parse(string) (Spec, error) +func NormalizeGitURL(string) (url string, githubRepository string, err error) +func NormalizeArchiveURL(string) (string, error) +func NormalizeSubpath(string) (string, error) +``` + +`Repository` remains populated only for the existing canonical HTTPS GitHub shorthand identity. Generic URL identity lives in `URL`; do not place a GitLab/Azure/SSH endpoint into a field interpreted by Directory as GitHub owner/repo. + +Accept these forms: + +| Input | Normalization / intent | +|---|---| +| Existing `[github:]owner/repo@FULL_SHA[//path]` | Preserve current exact-SHA interpretation and canonical/source-key bytes | +| `github:owner/repo[@REF][//path]` | Explicit Git shorthand; missing ref means default branch; bare `owner/repo` remains Directory to avoid changing existing intent | +| `https://host/team/repo.git[#REF[//path]]` | Native Git HTTPS; optional exact commit/ref; no fragment means default branch | +| `git+https://host/path[#REF[//path]]` | Explicit arbitrary HTTPS Git, including clone endpoints without `.git` | +| `ssh://[user@]host[:port]/team/repo[.git][#REF[//path]]` | Native SSH Git; preserve case-sensitive repository path | +| `[user@]host:team/repo[.git][#REF[//path]]` | SCP-like SSH Git; require a real host and nonempty safe path; accept Azure `git@ssh.dev.azure.com:v3/org/project/repo`; canonicalize spelling deterministically | +| `https://github.com/owner/repo[/tree/REF[/path]]` | Strip browser syntax, normalize to repository Git endpoint plus selector/path | +| `https://gitlab.com/group[/subgroups]/repo[/-/tree/REF[/path]]` | Support nested groups; `/-/tree/` separates repository from selector/path; same syntax on self-managed hosts containing that explicit marker | +| `https://dev.azure.com/org/[project/]_git/repo[?version=GBbranch|GTtag|GCsha&path=/package]` | Normalize clone endpoint; map GB/GT to explicit refs and GC to commit; decode path once | +| `https://org.visualstudio.com/[project/]_git/repo` | Azure legacy clone/browser host grammar; recognized query fields as above | +| HTTPS `.zip`, `.tar`, `.tar.gz`, `.tgz` | Complete archive source | +| HTTPS without recognized extension/provider/Git fragment | Bounded HTTPS body acquisition and sniffing; positive ZIP/TAR/gzip-TAR means archive. If the response is not an archive (including 401/404), try native Git for an arbitrary clone endpoint, then report a sanitized combined source error. No credential-bearing HTML/body logging | +| Explicit local archive path | Complete archive source; imperative local source unless represented explicitly in a local project manifest in the later project-scope PR | + +**Classification precedence is frozen:** (1) explicit local directory/archive paths and explicit git+https/ssh/SCP/Git shorthand or explicit Git selector; (2) recognized archive suffix or #sha256 archive selector, including GitHub/GitLab release URLs; (3) recognized provider repository/tree URL; (4) remaining HTTPS auto-classification. Conflicting explicit selectors/formats fail rather than overriding each other. An explicit archive suffix or positive ZIP/gzip/TAR magic recognition commits the operation to archive semantics: corrupt/truncated/oversize archive, malformed manifest and candidate-selection failures never fall back to Git. For extensionless HTTPS, fetch once without native credentials, retain the exact bytes/snapshot for package selection, and use native Git fallback only after a non-archive response/classification (including 401/404 or HTML). The probe never consults Git helpers or forwards credentials/headers. Multi-candidate archive diagnostics and any selected candidate use that one acquired byte stream. + +Provider tree selectors containing `/` require unambiguous handling. The pure parser retains provider and entire undecided suffix in transient `BrowserTree`; it does not guess Ref or Path. `ResolvedGit` belongs to sourceacquisition (alongside RefResolution), contains `{Git sourcespec.Git; Resolution RefResolution}`, and does not make the pure package depend on an adapter. Add network helper `ResolveGitSource(ctx, sourcespec.Git) (ResolvedGit, error)` in sourceacquisition: enumerate exact remote branch/tag names once, match possible URL suffix boundaries, and return a fully resolved Git with Browser=nil, exact Ref/Commit/Path, plus RefResolution. Accept only one matching selector/path decomposition; fail with `use the clone URL#refs/heads/branch//package` when more than one interpretation is valid. Full SHAs are unambiguous. Only resolved coordinates enter manifest/lock/source identity. File/blob browser URLs are rejected with a tree/package-root hint rather than importing one file. + +Use fragment `#sha256=<64 lowercase hex>[&path=]` for explicit archive checksum/path. Git `#REF//path` retains familiar legacy behavior; exact `refs/heads/...` / `refs/tags/...` avoid branch/tag ambiguity. Do not accept credentials/passwords/tokens in URLs. HTTPS userinfo is rejected (Azure username-only clone prefixes can be safely stripped before persisting/using the standard endpoint). SSH username is transport routing, preserved; passwords are rejected. Reject control characters, backslashes, unsafe encodings, empty/duplicate provider query fields and arbitrary query/fragment aliases. Recognized Azure `path`, `version`, `_a=contents` are consumed then dropped from endpoint identity. Do not lowercase generic repository paths or conflate SSH and HTTPS access principals; GitHub HTTPS aliases only map to existing GitHub identity. + +### 3.2 Additive identity, lock and intent + +Extend Git source structs with optional `URL` and make `Repository` optional, requiring exactly one except that canonical GitHub HTTPS normalization intentionally chooses the old Repository representation. Existing valid structures, source keys, canonical identity and manifest digests remain byte compatible. + +```go +GitSource{ Repository string, URL string, Path string, Selector Selector } +LockedGitSource{ Repository string, URL string, Path string, Requested Selector, + Commit string, ResolvedRef string, DefaultBranch string } +ArchiveSource{ URL string, Path string, SHA256 string } // SHA256 optional explicit byte pin +LockedArchiveSource{ URL string, Path string, RequestedSHA256 string, SHA256 string } +Requirement{ Alias string; Git *GitSource; Registry *RegistrySource; Archive *ArchiveSource } +LockedSource{ Git *LockedGitSource; Registry *LockedRegistrySource; Archive *LockedArchiveSource } +``` + +`ArchiveSource` is HTTPS only in portable saved dependencies. Local archive inputs follow existing local-directory imperative behavior, recorded with a sanitized absolute path and content checksum, and are not silently exported as portable URLs. All remote ready archives save by default and participate in closure resolution. Add `archive = "https://..."`, optional `sha256 = "<64 hex>"`, and `path = "..."` to `uap.toml`; reject mixed `archive/git/registry`, Git selectors on archives and checksum on Git. Git keeps current `git/ref/commit/path` syntax. + +Freeze new stable source keys as `git-url-v1:<64 lowercase sha256 hex>` and `archive-url-v1:<64 lowercase sha256 hex>`, where the hash preimage is canonical JSON encoding of the three-string array `[kind, canonical endpoint URL, logical package path]` with no optional elements. Use this one pure helper in sourcespec for Requirement.SourceKey, lock validation and new SourceBindingID inputs. Snapshot-cache keys use the same tuple source key plus immutable commit/archive checksum, digest algorithm and tree digest. Existing GitHub and Directory source keys are untouched. Checksum/commit/selector are version facts, excluded from stable source ownership. Preserve generic path case, explicit SSH user, URL port and decoded portable path meaning; tests cover URL delimiters and case-distinct paths. Add a typed archive selector `SelectorArchive` whose value is the explicit SHA256 or empty for a moving URL; its validator accepts only empty or one lowercase hex SHA256. Root and transitive graph selectors, edge validation, source reconstruction and update policy must handle all three variants explicitly instead of relying on else = Registry. + +Keep manifest schema and lock version 1 as additive manager fields: old binaries already reject unknown manifest/lock fields fail closed. Confirm by tests that no old field changes meaning and old locks retain existing normalized output. If review determines a version bump is required, use a new lock version only for new source variants and continue accepting old locks without rewriting identity. + +Add optional source provenance fields `SourceKind`, `SourceURL`, `ArchiveSHA256` to both `domain.SourceIdentity` and `SourceBinding`; copy them through every state/store/usecase projection. Source kind is `git`/`archive`; zero values keep historical source data. For generic Git, `ResolvedRevision` remains the exact commit; for archives use a clearly tagged `sha256:` immutable revision. New canonical direct identities are stable endpoint+package identities, with exact version separate. Preserve historical direct GitHub canonical identity bytes. + +Compute source binding identity from stable endpoint/package for new types. Old GitHub dependency special-case behavior stays unchanged. Updating a moving URL/ref must not change ownership; switching repository/URL/package still requires explicit switch/rebind. Repair always uses the installed binding's exact commit or archive checksum, never today's moving branch/download. + +### 3.3 Acquisition interface and compatibility + +Retain the existing `SourceAcquirer` interface and GitHub methods for signed callers and existing test fakes. Introduce a separate optional `DirectSourceAcquirer` capability implemented by `sourceacquisition.Acquirer` and the lazy composition wrapper: + +```go +ResolveGitSource(context.Context, sourcespec.Git) (ResolvedGit, error) +ResolveGit(context.Context, sourcespec.Git, selector string, exactRef bool) (RefResolution, error) +DiscoverGitPackages(context.Context, sourcespec.Git, commit string) ([]string, error) +AcquireGit(context.Context, sourcespec.Git, commit, expectedTreeDigest string) (domain.PackageSnapshot, error) +AcquireArchive(context.Context, sourcespec.Archive, expectedArchiveSHA256, expectedTreeDigest string) (domain.PackageSnapshot, error) +``` + +AcquireGit requires an immutable commit. Selector resolution occurs once per new/update intent before discovery/loading. Signed Directory/Discovery continue calling existing `AcquireGitHubVerified`, without user credential mode or generic source parsing. Use one CLI helper to dispatch GitHub old/new methods and one helper to construct identity from a locked package; every restore path uses them rather than manually rebuilding `githubIdentity`. + +Archive acquisition returns byte digest and selected **logical** package path in snapshot source metadata. Physical extraction wrapper names never enter source identity or portable pins. The archive adapter does not parse standard plugin schema or evaluate package code. CLI validation uses the standard loader and the existing security/consent boundary. A missing optional direct capability produces a precise capability error in old fakes, not panic or accidental Directory fallback. + +### 3.4 Native Git and credential boundary + +Keep current anonymous isolated OSRunner behavior for signed GitHub. Add a per-command transport mode used only by direct user-selected Git `ls-remote`/`fetch` operations. These operations run in a fresh private repository, outside the caller's checkout, using the user's HOME/USERPROFILE/XDG Git credential configuration, SSH config/known-hosts and SSH_AUTH_SOCK, plus trusted authentication environment variables (`GIT_ASKPASS`, `SSH_ASKPASS`, `GIT_SSH`, `GIT_SSH_COMMAND`, `GIT_SSH_VARIANT`, CA/proxy inputs as appropriate). Do not persist credential material or copy it into package snapshots. Preserve process-tree cleanup. + +Strip ambient Git repository/object/index/worktree overrides, tracing destinations, arbitrary `GIT_CONFIG_COUNT/KEY/VALUE` injection and unsafe TLS bypass. Override hooks location to an empty private directory, disable fsmonitor/maintenance/submodule recursion and disallow non-HTTPS/SSH protocols. `GIT_TERMINAL_PROMPT=0`, SSH batch mode and `GCM_INTERACTIVE=Never` ensure automated commands do not hang; existing helpers/agent may supply already provisioned credentials. Do not disable host-key checking or TLS verification. Authentication errors explain how to configure native Git credentials without echoing native stderr, URLs with secrets, helper output or command environment. + +**Crucial checkout separation:** direct transport fetches full blobs for the shallow immutable commit (omit blob:none). Checkout/materialization then uses the current anonymous isolated environment and no user/system filter/hook configuration. This prevents a fetched `.gitattributes` from triggering a user-installed smudge filter or LFS downloader and avoids a credential-dependent lazy fetch during isolated checkout. Configure safe isolated repo settings before fetch/checkout; package checkout stays detached, sparse paths remain validated and gitlinks/symlinks retain existing source snapshot policy. Do not run a generic ambient `git clone` followed by checkout. + +Resolve annotated tags to commits and verify fetched commit. Default/ref update uses exact remote selection with no broad refspec or revision expression. Root discovery shares native Git transport and remains bounded. Any credential-related state stays in the owned temporary repository and is removed on success, error, declined confirmation and cancellation. + +### 3.5 Archive adapter policy + +Archive implementation can be a separate worker after both plan reviews; its owned files are new `sourceacquisition/archive*.go` only (including tests). Source lead owns sourcespec and domain shapes and supplies the agreed API. No edits to acquirer.go, environment or CLI by the archive worker. + +* HTTPS only for remote archives; standard certificate verification, bounded redirects (max 5), HTTPS at every hop, no URL userinfo, timeout/context cancellation, no response-body/error dumping. No supplied authorization header/cookie forwarding. Extensionless downloads use magic/header inspection; content type is advisory, not authority. Query-bearing signed-token URLs are rejected to keep credentials out of persistent intent/locks; stable public endpoints are the supported package URL contract. +* Max compressed/download bytes 256 MiB, max archive entries 10,000, max regular file 64 MiB, max total expanded bytes 256 MiB, max depth 64. Limits apply to declared and actual data before writing/allocating; ZIP central-directory bounds do not replace streamed counters. Bound gzip expansion and consume/validate the complete archive including checksum/truncation errors and trailing/concatenated content policy. +* Support ZIP (store/deflate only, reject encryption/unsupported methods), TAR, gzip TAR. Detect incompatible format/suffix and corrupted streams; never treat HTML, a bare SKILL or plugin.json as an archive. Reject multiple ZIP volumes and malformed/truncated headers. +* **Stream termination is frozen:** accept one archive only. ZIP must begin at byte zero with a local-file header (or an empty EOCD, which subsequently fails package selection), have valid single-disk central-directory/end records and terminate exactly at the EOCD plus its declared bounded ZIP comment. No preamble/self-extracting prefix, appended archive or undeclared trailer is accepted. ZIP64 is unsupported in this first delivery because all accepted entry/count/byte sizes fit ordinary ZIP; reject its records explicitly with a format-limit diagnostic. TAR requires its two zero end blocks, followed only by whole 512-byte zero padding blocks within byte limits; reject a second TAR or nonzero/misaligned tail and missing/truncated end blocks. Gzip TAR is exactly one gzip member containing the accepted TAR stream, with CRC/size/trailer validation; disable concatenated-member mode and reject any extra compressed member or outer trailing bytes. Conventional in-member TAR zero padding remains accepted and bounded. Test these precise boundaries, valid padding/comments, CRC/truncation, concatenated gzip/TAR and prefix/trailer payload. +* Adapt retained TAR policy: normalized NFC portable relative paths; reject traversal/absolute/drive/UNC/backslash/control paths, links, devices/special entries, duplicate names, Unicode/case collisions including implicit ancestors, file/directory conflicts, excessive declared sizes and archive bombs. Ignore benign `./` root directory records only if defined explicitly in tests; do not normalize traversal into acceptance. TAR PAX metadata is validated through resulting headers; hard links and symlinks are rejected. +* Extract only to newly created private `0700` temp root via rooted/no-follow exclusive file operations. Strip special bits; retain only ordinary executable intent as metadata for `SnapshotWithExecutables`, consistent across OSes. No overwrite, chmod of external paths or running extraction utilities. +* Logical archive root is the extraction root when it has root plugin.json. Otherwise, if every archive member lies under exactly one outer directory (no sibling files), strip exactly that one physical wrapper before package-path selection, even when its package is nested. This wrapper is transient and never persisted. Explicit Path is relative to this logical root; empty Path prefers its root plugin.json, then bounded package-shaped manifest candidates. One candidate may be selected; multiple candidates require explicit Path and are listed without a second download. The standard loader validates only the selected package. A malformed logical root manifest fails without nested fallback. Updating a wrapper from repo-old/ to repo-new/ preserves logical path and stable identity. The adapter API returns the logical path it selected, never an extracted absolute path or wrapper name. +* Compute SHA256 of original archive bytes; check requested/locked byte checksum before extracting. Re-seal selected package with packagedigest and check locked tree digest. Lock both byte checksum and tree/manifest digests. If the endpoint changes bytes with identical extracted tree, locked restore still fails byte identity unless the verified store already has the original tree. Update may adopt changed bytes only by the normal explicit update/consent flow. +* Return cleanup-owned sealed snapshot; remove download and extraction roots on all paths. Never publish store/state/manifest before existing consent and commit boundaries. + +## 4. Implementation components and exact ownership + +| Component | Owned production files | Review focus / outputs | +|---|---|---| +| S1 pure grammar | new `install/integrationctl/agentplugins/sourcespec/{source,git,provider,archive}.go` and focused tests | provider grammar, path/ref ambiguity, redaction, canonicalization and old input preservation | +| S2 native Git | `.../adapters/sourceacquisition/{acquirer,refs}.go`, new `git_transport.go`, `git_direct.go`, relevant focused tests | separate anonymous/authenticated transport; isolated checkout; exact revisions, limits, cleanup | +| S3 archive | new `.../adapters/sourceacquisition/archive*.go` only | safe downloader/parser/extractor, original byte digest, root selection, real ZIP/TAR/gzip fixtures | +| S4 environment contract | `.../environment/{model,graph,resolve_graph,resolve_expand,coordinator}.go` as necessary; new `sources.go`; `.../adapters/dependencymanifest/decode.go`; `cli/internal/agentpluginscli/manifestedit/edit.go`; associated tests | three typed variants, stable identities, transitive graph/edge validation, backward-compatible lock/TOML encoding | +| S5 lifecycle integration | `cli/internal/agentpluginscli/{app,source,saved_add,deps_resolver,deps_staging,deps_sync,environment_session,lifecycle,add_multi}.go`; new direct-source helpers; `cli/cmd/agentplugins/main.go` lazy wrappers/composition where needed; `domain/{types,state,identity}.go`; `usecase/{installation,binding,group_commit}.go` explicit source copies | saved/default/--no-save paths; immutable repair; update/switch identity; cache dispatch; no Directory relaxation; provenance output | +| S6 E2E/docs | new focused source journeys in `cli/internal/agentpluginscli/*source*e2e_test.go`; native transport fixture helpers; `docs/UAP_DEPENDENCIES.md`; small source guide `docs/PLUGIN_SOURCES.md`; relevant README/website command guidance only after implementation verified | actual native transports plus CLI lifecycle; truthful support table and credential setup; no release qualification claims | + +On every content-store cache hit, including deps_candidates.go, loadStoredDependency, materializeLockedPackage, DescribePinned and replay/repair paths, reconstruct SourceIdentity from the validated selected locked package before loading or computing binding identity. Store.Metadata.Source is historical first-writer provenance only: two endpoints or two byte-distinct archives may share one tree entry. Do not copy per-source trees, rewrite metadata or adopt cached ownership. Test both population orders for same-tree distinct generic Git/archive origins and distinct archive byte pins. + +Do not modify `directoryv1` signatures/parser/policy for new direct types. Replace the new direct call sites' dependency on `ResolveDirectExact` with pure source validation; historical anonymous GitHub paths retain their current checks. Store layout and standard loader need no new feature-specific branches. Project scope worker owns scope resolution/client capability work. Source helpers consume the already selected App and environment context, including project scope in Directory policy requests; they never reconstruct a user App during replay. Exchange App/model/coordinator touchpoints before edits; source changes are integrated above project core before the project source-seam scenario. Do not independently edit the project worktree. + +## 5. Meaningful verification and E2E matrix + +Unit/contract tests cover grammar, malformed/ref ambiguity, URL credentials, exact source key stability, backward old fixtures, source clone/copy persistence, lock edges/variant mixtures, archive hostile inputs and cancellation. Fake-runner tests cover command construction and environment policy but are not called transport E2E. + +Native transport fixtures must execute the installed Git binary with no URLForRepo local-path substitution: + +1. Real public HTTPS Git over loopback TLS: Go HTTPS CGI handler invokes `git http-backend` against a disposable bare repository; per-test CA trust only. Use extensionless clone endpoint to exercise auto classification and `git+https` explicit path. +2. Private HTTPS Git: same native smart-HTTP server requires Basic auth; disposable HOME contains a test credential helper restricted to exactly the fixture host/path; failures without credentials, success with credential helper, no secrets in JSON/state/lock/store diagnostics. Assert no authenticated fallback/download to unrelated origin. +3. Real SSH Git: `/usr/sbin/sshd` on a loopback high port, isolated temporary host/user key and known_hosts, one forced-command `git-upload-pack` boundary limited to the disposable bare repo, no password/public network. Use native `ssh` and optionally a disposable ssh-agent socket. Test ssh:// port and SCP-like grammar separately where SCP spelling cannot encode a port (SSH config alias). Verify untrusted host refusal and cancellation tears down owned child processes. +4. HTTPS archives: serve real ZIP, TAR, TAR.GZ/TGZ and extensionless variants from loopback TLS with redirects. Lock -> remove source/store copy -> fresh environment restore from exact bytes; moved URL/repacked byte change must fail locked replay. Local archive add verifies source selection and cleanup without declaring portability. +5. Each supported source kind reaches the real standard loader, store, CLI plan/apply and existing disposable client adapter fixture. A standard skill-only package is enough; no package runtime or MCP server execution. Add payload files whose commands would create a sentinel if run and prove no sentinel exists after every phase. Security evaluation fixture stays local and inert; no external content scanner upload. +6. Lifecycle: add saves intent/lock, repeat no-op, moving ref/URL update keeps source ownership, installed repair uses old immutable pin after remote moves, source switch blocked, export/import into empty store, transitive generic Git/archive dependencies with shared target union, remove retains unrelated package/client files. One representative client for full transport cross product; reuse existing cross-client tests for shared lifecycle boundaries instead of multiplying transport/client matrices unnecessarily. +7. Directory and Discovery regression cases retain the anonymous runner, signed exact GitHub source and digests. New direct HTTPS/SSH/archive inputs never fall through to a Directory selector. Existing exact GitHub saved lock/source key tests retain their original values. +8. Dry run/decline/cancellation: no durable environment/store/projection changes; downloads, private Git temp dirs and request-local snapshots cleaned; at most one exact acquisition per missing pin per command. Offline status/tree use lock/store without contacting sources. Failed auth or mutated archive leaves manifest/lock/state unchanged. + +Fixture availability observed locally: git, ssh, sshd, openssl present. Use the root-installed Go 1.26.10 at `the Go 1.26.10 toolchain`. Standard CI should install openssh-server for a required Linux source-transport lane; local absence yields an explicit skip only outside that required lane. Add a small opt-in network smoke against existing public provider repositories if necessary, but do not let it substitute for deterministic native Git fixtures or upload package content. + +## 6. Checkpoints and review sequence + +1. Source plan R1 by project_core completed at `sources-review-r1.md`; this revision fixes mandatory identity-from-lock on all cache paths, wrapper-independent archive logical paths, transient browser suffix/resolved-path API, one versioned tuple-hash source-key encoding and selected-App/project integration. Reviewer verification pending; no code. +2. Root sends revised plan to `client_plans` for source plan R2. Fix findings and freeze API/file ownership. No code until root explicitly acknowledges both rounds complete. +3. S1/S4 typed model and grammar checkpoint, focused contracts; local commit. S3 archive adapter may proceed in disjoint files after contracts are shared. +4. S2 native Git checkpoint, real transport tests; local commit. +5. S5 saved/lifecycle/restore integration checkpoint, focused journeys; local commit. +6. Merge archive worker commits, full source journey fixtures/docs; local commit. +7. Two implementation review/fix rounds according to root routing; required focused/full module gates and E2E on exact candidate. Send SHAs, changed files, test commands/results, limits and unresolved items to root. Root performs all pushes/PR publication. + +## 7. External primary references checked + +* Git official environment/configuration documentation: https://git-scm.com/docs/git and https://git-scm.com/docs/git-config . HOME/global/system config and GIT_CONFIG_COUNT need explicit treatment; GIT_SSH_COMMAND is shell-interpreted and is trusted user configuration, never synthesized from source text. +* Git credential helpers: https://git-scm.com/docs/gitcredentials.html . Existing configured helper programs are native authentication infrastructure; helper content/output is not package content and must not be serialized/logged. +* Azure clone URL documentation: https://learn.microsoft.com/en-us/azure/devops/repos/git/clone?view=azure-devops . Clone URLs need not end `.git` and commonly include a username-only organization prefix; public/private native Git authentication belongs to Git rather than package metadata. +* GitLab repository documentation / live first-party tree grammar: https://docs.gitlab.com/user/project/repository/ and https://gitlab.com/gitlab-org/gitlab/-/tree/master/doc . `/-/tree/` is a clear repository/browser separator; nested subgroup paths must remain intact. + +Azure browser query mappings and provider branch/path ambiguity tests must additionally be verified against provider-owned examples/source while implementing S1; no unsupported query interpretation should be guessed merely to make a fixture pass. + +## R1 fix ledger + +* Shared tree cache provenance: obligatory identity-from-selected-lock on every store load, including candidate loading, plus both cache population orders and byte-distinct/same-tree archives. +* Archive wrapper identity: logical root strips at most one common physical wrapper; Path is logical and stable across changing version wrappers. Selection does not require an archive-to-standard-loader dependency or a second download. +* Provider parse/resolution: BrowserTree carries undecided suffix without I/O; ResolveGitSource returns resolved final Path and immutable commit/ref. +* Source key/seam: fixed versioned SHA256 of canonical three-string JSON tuple; old keys exact; source lane consumes selected App and integrates above scope core. +* Additional self-review correction: SCP-like SSH accepts Azure SSH endpoints without .git suffix; shell/control/path ambiguity stays rejected. + +## R2 fix ledger + +* Archive stream termination frozen to one non-ZIP64 ZIP with no prefix/undeclared trailer, one TAR with two end blocks + whole zero padding, or one gzip member containing that TAR; CRC/size/end/trailing validation is mandatory and bounded. +* Classification order frozen: explicit transport/selectors, archive shapes, provider Git URLs, then HTTPS auto. Recognized archives never fall back to Git; one credential-free HTTPS byte acquisition feeds selection. Native fallback is limited to non-archive responses. +* ResolvedGit explicitly belongs to sourceacquisition, keeping sourcespec pure and free of RefResolution/adapter imports. +* Focused regressions added: provider-hosted release archives; valid/corrupt extensionless archives; private extensionless clone; HTML then Git failure; multi-candidate single-request archive; no helper/auth forwarding in probe. diff --git a/docs/plans/source-transports-e2e/sources-review-r1.md b/docs/plans/source-transports-e2e/sources-review-r1.md new file mode 100644 index 000000000..cf15dfb47 --- /dev/null +++ b/docs/plans/source-transports-e2e/sources-review-r1.md @@ -0,0 +1,130 @@ +# Independent sources plan review — round 1 + +Reviewer: project-core lane. Reviewed `sources-plan.md` against implementation +base `3ea24e3014ddf6679ca86729c56f1069112b0d3a`, including current source +acquisition/ref resolution, content store, dependency staging/candidate loading, +source-binding identity, and graph pin/edge validation. No implementation. + +Verdict: **changes required before implementation**. The one engine/store, +typed additive source variants, native credential isolation and checkout +separation, signed Directory boundary, archive limits, and native transport E2E +direction are accepted. The remaining issues are narrow contracts, not grounds +for another architecture. + +## P1 — shared tree cache must not lend another source its provenance + +The existing immutable store is keyed only by tree digest. `Store.Put` returns +an existing entry without changing/comparing source metadata; `Metadata.Source` +describes whichever source populated that digest first. `deps_candidates.go` +currently loads that metadata as the package Source. With new Git/archive +variants, the exact same tree may come from two endpoints or two byte-distinct +archives. Trusting cached source provenance can manufacture the wrong binding +ID, requested source, archive checksum or installed repair coordinates even +though package bytes correctly match. + +The plan mentions one identity-from-lock helper; make its use obligatory on +**every** cache hit/load/replay path, including existing store candidate loading, +not just missing-pin downloads. Treat stored source metadata as historical +provenance only. Reconstruct current immutable source identity from the validated +selected locked package; keep tree/executable verification against the content +store. Do not add per-source copies of identical trees or rewrite metadata to +the latest caller. + +Required regression: install the same tree from two distinct generic Git or +archive sources, in both cache population orders; the store remains shared, +and each lock/installation/binding/repair operation retains its own endpoint, +subpath, commit or byte checksum. Also use two different archive byte streams +with the same extracted tree and verify their separate locked byte pins. + +## P1 — archive wrapper names must not become moving source identity + +The archive selection policy says to strip one outer wrapper and return the +selected package path in snapshot metadata, while ownership uses URL + package +path and locks persist Path. A normal provider archive wrapper often changes +with the commit/version. If a discovered physical wrapper name becomes Path, +updating the same URL/package appears to switch source, or locked restore looks +for a path in the wrong logical archive root. + +Define logical package path independently of extraction wrappers and temporary +directories. Specify precedence and meaning for an explicit Path versus a +detected wrapper: normalize one deterministic wrapper root, then select a +logical package path relative to it, or preserve an explicit documented raw-root +path mode. The chosen model must keep identical package identity across +`repo-old/` -> `repo-new/` archive updates. Do not persist extracted absolute paths +or a version-derived wrapper as canonical package identity. + +Required regression: moving archive URL changes wrapper name and package bytes; +normal update retains source/binding ownership, exact locked replay still checks +the original archive checksum/tree, and an explicit logical nested package path +selects the same package under either wrapper. Malformed logical root manifests +must still block nested fallback. + +## P2 — pure provider parsing needs an explicit deferred tree-selector shape + +`sourcespec.Parse` is pure, but browser tree URLs with slash-containing refs need +one native remote-ref enumeration to identify selector/path boundaries. The +proposed `Git{URL,Path,Ref,Commit}` has only resolved coordinates and the direct +interface `ResolveGit(...Git, selector, exactRef)` returns a commit/ref without a +resolved package Path. There is no agreed carrier for an ambiguous raw suffix or +the resulting package path. + +Keep the existing pure package. Add the smallest transient representation for +unresolved browser suffix and provider grammar, plus one network-layer helper +that returns a fully resolved Git spec and immutable revision. Alternatively +resolve explicitly in the CLI before constructing final `sourcespec.Git`. +Declare that boundary and its output shape in the plan/API so neither the pure +parser nor archive worker guesses the first URL segment as the ref. Only final +resolved coordinates enter manifest/lock/source identity. Tests must cover one +unambiguous slash branch and two valid ref/path decompositions rejected with the +clone-URL hint. + +## P2 — finalize source-key encoding and cross-lane integration seam + +The plan leaves source-key encoding as structured escaping **or** JSON hash. +Freeze one before S1/S4 and archive worker implementation. A compact canonical +tuple encoded with an unambiguous delimiter/escaping rule or a versioned hash +is sufficient; preserve old GitHub keys exactly. Use identical canonicalization +in Requirement.SourceKey, locked validation, SourceIdentity/SourceBinding and +snapshot-cache keys. Test delimiter-containing valid URL/path inputs and generic +path case, including old GitHub canonical keys. + +Coordinate `App` and environment model/coordinator edits with project core: +source helpers must consume the selected App/environment context, preserve +explicit project scope in Directory policy requests, and never rebuild a user +App during replay. Sources remain stacked above project core. New source facts +must also survive copies of a project binding through repair/remove/recovery; +one source journey in a qualified project fixture is enough beyond the full +transport matrix's representative user fixture. + +## Accepted boundaries and verification + +* Preserve existing anonymous signed GitHub methods and old fake interface; + additive direct capability is appropriately bounded. +* Fetch with native credentials, then materialize with isolated config and + complete blobs. No ambient clone/checkouts or legacy engine dependency. +* Query token rejection and no stderr/environment logging keep source intent, + locks and diagnostics free of authentication material. +* Complete-package archive extraction, no-follow exclusive writes, streamed + expansion/entry limits, executable inventory and byte/tree digest checks fit + existing standard snapshots. No plugin execution is needed for validation. +* Real HTTPS smart Git/private helper/SSH and HTTPS archive fixtures exercise + transports beyond fake-runner command construction. Reuse one full client + lifecycle, then one project-context seam scenario instead of an unnecessary + transport-by-client cross product. +* Old locks, selectors, manifest digest and canonical source IDs stay exact; + additive formats fail closed in old readers. Preserve signed Directory + eligibility rather than using new transport support as endorsement. + +No style-only requests. Parent still owns second independent review and the +implementation authorization gate after fixes. + +## Fix verification — 2026-10-23 + +Verified the revised sources plan contains mandatory identity reconstruction +from the validated selected lock on all store/candidate/replay paths; +wrapper-independent logical archive roots and package paths; the transient +BrowserTree plus final resolved Git coordinates; one versioned tuple-hash key +encoding; and selected-App/project lifecycle integration. All R1 findings are +closed. **R1 GO**, with parent authorization and independent R2 still required +before implementation. Preserve the requested changing-wrapper update and +explicit nested logical-path regression during implementation verification. diff --git a/docs/plans/source-transports-e2e/sources-review-r2.md b/docs/plans/source-transports-e2e/sources-review-r2.md new file mode 100644 index 000000000..31818eee2 --- /dev/null +++ b/docs/plans/source-transports-e2e/sources-review-r2.md @@ -0,0 +1,85 @@ +# Independent sources plan review — round 2 + +Reviewer: client-plans lane; 2026-10-23. Reviewed the post-R1 source plan, +`sources-review-r1.md`, current acquisition/runner, source identity/binding, +saved-add and dependency staging code. Official Git environment/configuration +references were rechecked. No implementation performed. + +Verdict: **two small contract fixes required; architecture and R1 fixes GO**. +These do not justify new abstractions, a second engine, or broader test matrices. + +## P2 — freeze archive stream termination before assigning the archive lane + +The plan requires validation of trailing/concatenated content but still calls +it a policy rather than defining one. Go gzip normally permits concatenated +members, and TAR readers can report end-of-archive before consuming a second +stream. Without a frozen rule, a worker can legitimately checksum bytes that +the extractor never assessed, while another worker assumes every member was +validated. This is an input-contract gap, not a demand for more formats. + +**Fix:** select one strict first-delivery policy: one ZIP archive; one TAR stream +with only defined zero block padding after its end marker; one gzip member +containing that TAR stream, rejecting an additional member or non-padding +trailing payload. Define any ZIP preamble/trailer handling explicitly rather +than accept self-extracting/polyglot containers by accident. Bound all retained +padding by the existing byte limits. The archive lane tests exactly these end +conditions, CRC/truncation, valid conventional padding, concatenated gzip/TAR, +and payload trailing an otherwise valid archive. Equivalent equally strict +rules are acceptable if frozen before implementation. + +## P2 — define source-recognition and auto-classification precedence + +The grammar table includes provider browser URLs, Git `.git`/fragments, +recognized archive suffixes, and HTTPS sniffing. It does not state recognition +order or when classification becomes irrevocable. A provider-hosted release +archive must not be parsed as a browser repository; a positively recognized +but malformed archive must not be retried as Git. Conversely a private +extensionless Git endpoint's 401 must reach native Git credential handling +without leaking authentication to the preliminary downloader. + +**Fix:** freeze parser precedence and the auto-classification decision tree: +explicit source scheme/selector first; recognized complete archive shape before +provider repository/tree shape; all other HTTPS is bounded auto-classification. +After positive archive identification, corruption/manifest/selection errors +stay archive errors with no Git fallback. For archive auto-detection, preserve +the acquired bytes/snapshot across candidate selection so no second download +can change the selected revision. Fallback native Git only after a non-archive +response/classification, with the existing sanitized errors and credential-free +HTTPS downloader. The exact ordering can differ if it has the same unambiguous +outcomes and is documented. + +Required focused regressions: GitHub/GitLab-hosted release archive; +extensionless valid and corrupt archive; private extensionless clone endpoint; +HTML rejection then native Git failure; multi-candidate archive selection with +one HTTP request; and no helper/auth/header forwarding during the HTTP probe. + +## Accepted components + +| Component | Assessment | +| --- | --- | +| One pure source grammar + transient browser suffix | GO; parsing remains offline and final selector/path comes from one network-layer resolver. | +| Generic URL/archive typed source variants | GO; three-way validation explicit, legacy GitHub bytes and signed Directory boundary preserved. | +| Frozen versioned tuple-hash source keys | GO; stable ownership separates endpoint/package from commit/checksum. | +| Lock and state source-copy seam | GO; all immutable source facts must survive repair/remove/replay. Additive version remains contingent on the stated old-reader fail-closed test. | +| R1 cache provenance correction | GO; current source identity always comes from the selected validated lock, while one tree entry stays shared. | +| R1 wrapper-independent archive paths | GO; logical paths remain stable when physical version wrapper changes. | +| Native credentials for fetch only | GO; preserve trusted native authentication, isolate checkout, no package filters/hooks/remote helpers beyond HTTPS/SSH protocols. | +| Archive limits/paths/executables/cleanup | GO after termination/classification fixes above; byte pin precedes extraction and tree pin follows standard sealing. | +| Local archive behavior | GO; explicit current local-source imperative semantics, no false portable saved URL. | +| User/project selected App integration | GO; no reconstruction of user environment during replay; one project lifecycle seam is sufficient. | +| Native transport tests | GO; real Git smart HTTP/private helper/SSH and actual archive formats prove transports, with fixtures clearly separate from real-client/native runtime proof. | + +No unrelated native-client matrix expansion, scanner work, release, or provider +endorsement is needed. Send the revised contract for fix verification; parent +retains final implementation authorization. + +## Fix verification — 2026-10-23 + +Rechecked the revised source plan. Classification precedence is explicit; +positive archive recognition is irreversible; one credential-free download is +retained across selection. The archive policy now freezes one non-ZIP64 ZIP +with exact end/comment bounds, TAR end markers plus bounded aligned zero +padding, and one gzip member with validated trailer and no extra payload. +`ResolvedGit` belongs to the acquisition layer, preserving pure grammar. +Both findings are addressed. **R2 GO.** Parent authorization is still required +before implementation; no code was written during either review. From 391f3b8bc412efe9e0492e2c58dada100cc15df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Fri, 23 Oct 2026 18:29:25 +0300 Subject: [PATCH 02/25] feat(sources): define normalized direct source and archive acquisition contracts --- .../adapters/sourceacquisition/acquirer.go | 3 + .../adapters/sourceacquisition/direct.go | 33 ++++ .../agentplugins/domain/state.go | 3 + .../agentplugins/domain/types.go | 3 + .../agentplugins/sourcespec/git.go | 151 ++++++++++++++++++ .../agentplugins/sourcespec/parse.go | 84 ++++++++++ .../agentplugins/sourcespec/provider.go | 133 +++++++++++++++ .../agentplugins/sourcespec/source.go | 148 +++++++++++++++++ .../agentplugins/sourcespec/source_test.go | 78 +++++++++ 9 files changed, 636 insertions(+) create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/direct.go create mode 100644 install/integrationctl/agentplugins/sourcespec/git.go create mode 100644 install/integrationctl/agentplugins/sourcespec/parse.go create mode 100644 install/integrationctl/agentplugins/sourcespec/provider.go create mode 100644 install/integrationctl/agentplugins/sourcespec/source.go create mode 100644 install/integrationctl/agentplugins/sourcespec/source_test.go diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go index 191f2b01b..d0cfec707 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "fmt" + "net/http" "os" "path" "path/filepath" @@ -96,6 +97,8 @@ func isolatedGitEnvironment(environ []string, isolatedHome string) []string { } type Acquirer struct { + // HTTPClient is optional test/host transport configuration; redirects still obey archive policy. + HTTPClient *http.Client TempRoot string Runner Runner Digester packagedigest.Builder diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/direct.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/direct.go new file mode 100644 index 000000000..236bf0aaa --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/direct.go @@ -0,0 +1,33 @@ +package sourceacquisition + +import ( + "context" + "errors" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +// ResolvedGit contains final persistable coordinates. The pure source grammar +// never imports acquisition or performs remote-ref discovery. +type ResolvedGit struct { + Git sourcespec.Git + Resolution RefResolution +} + +// DirectAcquirer is separate from the signed GitHub source contract. AcquireGit +// requires a full commit; archive pins bind original bytes as well as tree bytes. +// Archive implementers own only archive*.go and return sealed cleanup-owned +// snapshots, with logical Path and ArchiveSHA256 in Source metadata. +type DirectAcquirer interface { + ResolveGitSource(context.Context, sourcespec.Git) (ResolvedGit, error) + ResolveGit(context.Context, sourcespec.Git, string, bool) (RefResolution, error) + DiscoverGitPackages(context.Context, sourcespec.Git, string) ([]string, error) + AcquireGit(context.Context, sourcespec.Git, string, string) (domain.PackageSnapshot, error) + AcquireArchive(context.Context, sourcespec.Archive, string, string) (domain.PackageSnapshot, error) +} + +// ErrNotArchive permits HTTPS auto-classification to try native Git only when +// no archive signature was recognized. Corrupt/oversize recognized archives +// must never return this sentinel. +var ErrNotArchive = errors.New("HTTPS response is not a package archive") diff --git a/install/integrationctl/agentplugins/domain/state.go b/install/integrationctl/agentplugins/domain/state.go index b764b6947..8ecf2935b 100644 --- a/install/integrationctl/agentplugins/domain/state.go +++ b/install/integrationctl/agentplugins/domain/state.go @@ -133,6 +133,9 @@ const ( ) type SourceBinding struct { + SourceKind string `json:"source_kind,omitempty"` + SourceURL string `json:"source_url,omitempty"` + ArchiveSHA256 string `json:"archive_sha256,omitempty"` SourceBindingID string `json:"source_binding_id"` RequestedSource string `json:"requested_source"` CanonicalSource string `json:"canonical_source"` diff --git a/install/integrationctl/agentplugins/domain/types.go b/install/integrationctl/agentplugins/domain/types.go index 28646fa94..00c73b5a2 100644 --- a/install/integrationctl/agentplugins/domain/types.go +++ b/install/integrationctl/agentplugins/domain/types.go @@ -42,6 +42,9 @@ type Diagnostic struct { } type SourceIdentity struct { + SourceKind string `json:"source_kind,omitempty"` + SourceURL string `json:"source_url,omitempty"` + ArchiveSHA256 string `json:"archive_sha256,omitempty"` RequestedSource string `json:"requested_source,omitempty"` CanonicalSource string `json:"canonical_source,omitempty"` Repository string `json:"repository,omitempty"` diff --git a/install/integrationctl/agentplugins/sourcespec/git.go b/install/integrationctl/agentplugins/sourcespec/git.go new file mode 100644 index 000000000..e95554398 --- /dev/null +++ b/install/integrationctl/agentplugins/sourcespec/git.go @@ -0,0 +1,151 @@ +package sourcespec + +import ( + "errors" + "net/url" + "strings" +) + +// NormalizeGitURL accepts only native HTTPS/SSH transports, never local paths, +// remote helpers, passwords or query aliases. SCP paths retain relative-path +// semantics instead of being changed into absolute ssh:// paths. +func NormalizeGitURL(value string) (string, string, error) { + if !safeText(value) { + return "", "", errors.New("unsafe Git endpoint") + } + if !strings.Contains(value, "://") { + left, right, ok := strings.Cut(value, ":") + if !ok || right == "" || strings.HasPrefix(right, "-") || strings.ContainsAny(left, " /?#") || strings.ContainsAny(right, "?#") || (!strings.Contains(left, "@") && !strings.Contains(left, ".")) { + return "", "", errors.New("Git endpoint must use HTTPS, ssh:// or user@host:path") + } + user, host, hasUser := strings.Cut(left, "@") + if hasUser { + if user == "" || host == "" || strings.Contains(host, "@") { + return "", "", errors.New("invalid SSH endpoint") + } + } else { + host = user + user = "" + } + if strings.ContainsAny(host, "[]: ") || strings.ContainsAny(user, ": ") { + return "", "", errors.New("invalid SSH endpoint") + } + for _, segment := range strings.Split(right, "/") { + if segment == "." || segment == ".." || segment == "" { + return "", "", errors.New("unsafe SSH repository path") + } + } + prefix := strings.ToLower(host) + if hasUser { + prefix = user + "@" + prefix + } + return prefix + ":" + right, "", nil + } + u, err := url.Parse(value) + if err != nil || (u.Scheme != "https" && u.Scheme != "ssh") { + return "", "", errors.New("Git endpoint must use HTTPS or SSH") + } + u, err = normalizedURL(value, u.Scheme) + if err != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { + return "", "", errors.New("Git endpoint cannot contain query or fragment") + } + if u.User != nil { + _, password := u.User.Password() + if password || u.User.Username() == "" || strings.ContainsAny(u.User.Username(), "@:/ ") { + return "", "", errors.New("Git endpoint cannot contain credentials") + } + if u.Scheme == "https" { + if !isAzureHost(u.Hostname()) { + return "", "", errors.New("HTTPS Git endpoint cannot contain user information") + } + u.User = nil + } + } + repository := "" + if u.Scheme == "https" && u.Host == "github.com" { + repository = strings.TrimSuffix(strings.TrimPrefix(u.Path, "/"), ".git") + if !repositoryPattern.MatchString(repository) { + return "", "", errors.New("GitHub repository must be owner/repo") + } + repository = strings.ToLower(repository) + u.Path = "/" + repository + ".git" + } + return u.String(), repository, nil +} + +func isAzureHost(host string) bool { + return host == "dev.azure.com" || strings.HasSuffix(host, ".visualstudio.com") +} + +func parseGit(value string) (Git, error) { + base, fragment, hasFragment := strings.Cut(value, "#") + if strings.Contains(fragment, "#") { + return Git{}, errors.New("invalid Git selector") + } + packagePath := "" + if before, after, ok := strings.Cut(fragment, "//"); ok { + fragment, packagePath = before, after + } + if before, after, ok := strings.Cut(base, ".git//"); ok { + if packagePath != "" { + return Git{}, errors.New("duplicate package path") + } + base, packagePath = before+".git", after + } + var err error + packagePath, err = NormalizeSubpath(packagePath) + if err != nil { + return Git{}, err + } + endpoint, repo, err := NormalizeGitURL(strings.TrimPrefix(base, "git+")) + if err != nil { + return Git{}, err + } + g := Git{URL: endpoint, Repository: repo, Path: packagePath} + if hasFragment { + fragment = strings.TrimPrefix(fragment, "ref=") + if err := ValidateRef(fragment); err != nil { + return Git{}, err + } + if ValidCommit(fragment) { + g.Commit = fragment + } else { + g.Ref = fragment + } + } + return g, nil +} + +func parseShorthand(value string) (Git, error) { + value = strings.TrimPrefix(value, "github:") + base, packagePath, _ := strings.Cut(value, "//") + repository, selector := base, "" + if index := strings.LastIndex(base, "@"); index >= 0 { + repository, selector = base[:index], base[index+1:] + if selector == "" { + return Git{}, errors.New("empty Git selector") + } + } + if !repositoryPattern.MatchString(repository) { + return Git{}, errors.New("GitHub source requires owner/repo") + } + g, err := parseGit("https://github.com/" + repository + ".git") + if err != nil { + return Git{}, err + } + g.Path, err = NormalizeSubpath(packagePath) + if err != nil { + return Git{}, err + } + if selector != "" { + if err := ValidateRef(selector); err != nil { + return Git{}, err + } + if ValidCommit(selector) { + g.Commit = selector + } else { + g.Ref = selector + } + } + return g, nil +} diff --git a/install/integrationctl/agentplugins/sourcespec/parse.go b/install/integrationctl/agentplugins/sourcespec/parse.go new file mode 100644 index 000000000..55c7ca7e5 --- /dev/null +++ b/install/integrationctl/agentplugins/sourcespec/parse.go @@ -0,0 +1,84 @@ +package sourcespec + +import ( + "errors" + "net/url" + "strings" +) + +// Parse recognizes direct inputs only. Directory names and local directories +// remain the caller's grammar; no filesystem existence probe is performed. +func Parse(raw string) (Spec, error) { + if !safeText(raw) { + return Spec{}, errors.New("invalid source input") + } + out := Spec{Requested: raw} + if strings.HasPrefix(raw, "github:") || (!strings.Contains(raw, ":") && strings.Contains(raw, "@")) { + g, err := parseShorthand(raw) + out.Kind, out.Git = KindGit, &g + return out, err + } + if strings.HasPrefix(raw, "git+https://") || strings.HasPrefix(raw, "ssh://") || (!strings.Contains(raw, "://") && strings.Contains(raw, "@") && strings.Contains(raw, ":")) { + g, err := parseGit(raw) + out.Kind, out.Git = KindGit, &g + return out, err + } + if !strings.HasPrefix(raw, "https://") { + if HasArchiveSuffix(raw) && (strings.HasPrefix(raw, ".") || strings.HasPrefix(raw, "/") || strings.ContainsAny(raw, "/\\")) { + out.Kind = KindLocalArchive + out.Archive = &Archive{URL: raw} + return out, nil + } + return Spec{}, errors.New("not a direct remote or archive source") + } + u, err := url.Parse(raw) + if err != nil { + return Spec{}, errors.New("invalid HTTPS source") + } + if strings.HasPrefix(u.Fragment, "sha256=") || HasArchiveSuffix(u.Path) { + a, err := parseArchive(u) + out.Kind, out.Archive = KindArchive, &a + return out, err + } + if u.Fragment != "" || strings.HasSuffix(u.Path, ".git") || strings.Contains(u.Path, ".git//") { + g, err := parseGit(raw) + out.Kind, out.Git = KindGit, &g + return out, err + } + if g, recognized, err := parseProvider(u); recognized { + out.Kind, out.Git = KindGit, &g + return out, err + } + endpoint, err := NormalizeArchiveURL(raw) + out.Kind = KindHTTPSAuto + out.Archive = &Archive{URL: endpoint} + return out, err +} + +func parseArchive(u *url.URL) (Archive, error) { + fragment := u.Fragment + copy := *u + copy.Fragment = "" + endpoint, err := NormalizeArchiveURL(copy.String()) + if err != nil { + return Archive{}, err + } + a := Archive{URL: endpoint} + if fragment != "" { + values, err := url.ParseQuery(fragment) + if err != nil || len(values["sha256"]) != 1 || !ValidSHA256(values.Get("sha256")) { + return Archive{}, errors.New("archive fragment requires sha256=FULL_SHA256 and optional path") + } + for key, value := range values { + if (key != "sha256" && key != "path") || len(value) != 1 { + return Archive{}, errors.New("invalid archive selector") + } + } + a.SHA256 = values.Get("sha256") + a.Path, err = NormalizeSubpath(values.Get("path")) + if err != nil { + return Archive{}, err + } + } + return a, nil +} diff --git a/install/integrationctl/agentplugins/sourcespec/provider.go b/install/integrationctl/agentplugins/sourcespec/provider.go new file mode 100644 index 000000000..ca8a54faa --- /dev/null +++ b/install/integrationctl/agentplugins/sourcespec/provider.go @@ -0,0 +1,133 @@ +package sourcespec + +import ( + "errors" + "net/url" + "strings" +) + +func parseProvider(u *url.URL) (Git, bool, error) { + host := strings.ToLower(u.Hostname()) + if isAzureHost(host) { + g, err := parseAzure(u) + return g, true, err + } + segments := strings.Split(strings.Trim(u.Path, "/"), "/") + if host == "github.com" { + if len(segments) < 2 { + return Git{}, true, errors.New("GitHub source requires owner/repo") + } + base := *u + base.Path = "/" + strings.Join(segments[:2], "/") + base.RawQuery = "" + base.RawPath = "" + endpoint, repo, err := NormalizeGitURL(base.String()) + if err != nil { + return Git{}, true, err + } + g := Git{URL: endpoint, Repository: repo} + if u.RawQuery != "" || u.ForceQuery { + return Git{}, true, errors.New("GitHub source cannot contain query parameters") + } + if len(segments) == 2 { + return g, true, nil + } + if len(segments) < 4 || segments[2] != "tree" { + return Git{}, true, errors.New("choose a repository or tree package root URL") + } + g.Browser = &BrowserTree{Provider: "github", Suffix: strings.Join(segments[3:], "/")} + return g, true, nil + } + marker := "/-/tree/" + if index := strings.Index(u.Path, marker); index >= 0 { + base := *u + base.Path = u.Path[:index] + base.RawPath = "" + base.RawQuery = "" + endpoint, repo, err := NormalizeGitURL(base.String() + ".git") + if err != nil { + return Git{}, true, err + } + if u.RawQuery != "" || u.ForceQuery { + return Git{}, true, errors.New("GitLab tree source cannot contain query parameters") + } + return Git{URL: endpoint, Repository: repo, Browser: &BrowserTree{Provider: "gitlab", Suffix: u.Path[index+len(marker):]}}, true, nil + } + if host == "gitlab.com" { + if len(segments) < 2 || strings.Contains(u.Path, "/-/") { + return Git{}, true, errors.New("choose a GitLab repository or tree package root URL") + } + copy := *u + copy.Path = strings.TrimSuffix(copy.Path, "/") + ".git" + copy.RawPath = "" + endpoint, repo, err := NormalizeGitURL(copy.String()) + return Git{URL: endpoint, Repository: repo}, true, err + } + return Git{}, false, nil +} + +func parseAzure(u *url.URL) (Git, error) { + segments := strings.Split(strings.Trim(u.Path, "/"), "/") + index := -1 + for i, s := range segments { + if s == "_git" { + if index >= 0 { + return Git{}, errors.New("invalid Azure repository URL") + } + index = i + } + } + if index < 1 || index+2 != len(segments) { + return Git{}, errors.New("Azure source requires an _git repository URL") + } + q, err := url.ParseQuery(u.RawQuery) + if err != nil { + return Git{}, errors.New("invalid Azure source query") + } + for key, values := range q { + if len(values) != 1 || (key != "path" && key != "version" && key != "_a") { + return Git{}, errors.New("unsupported Azure source query") + } + } + if q.Get("_a") != "" && q.Get("_a") != "contents" { + return Git{}, errors.New("choose Azure repository contents") + } + base := *u + base.RawQuery = "" + base.ForceQuery = false + base.Fragment = "" + endpoint, repo, err := NormalizeGitURL(base.String()) + if err != nil { + return Git{}, err + } + g := Git{URL: endpoint, Repository: repo} + g.Path, err = NormalizeSubpath(strings.TrimPrefix(q.Get("path"), "/")) + if err != nil { + return Git{}, err + } + version := q.Get("version") + if version != "" { + if len(version) < 3 { + return Git{}, errors.New("invalid Azure source version") + } + switch version[:2] { + case "GB": + g.Ref = "refs/heads/" + version[2:] + case "GT": + g.Ref = "refs/tags/" + version[2:] + case "GC": + if !ValidCommit(version[2:]) { + return Git{}, errors.New("Azure commit requires a full lowercase SHA") + } + g.Commit = version[2:] + default: + return Git{}, errors.New("unsupported Azure source version") + } + if g.Ref != "" { + if err := ValidateRef(g.Ref); err != nil { + return Git{}, err + } + } + } + return g, nil +} diff --git a/install/integrationctl/agentplugins/sourcespec/source.go b/install/integrationctl/agentplugins/sourcespec/source.go new file mode 100644 index 000000000..3ba9ccb12 --- /dev/null +++ b/install/integrationctl/agentplugins/sourcespec/source.go @@ -0,0 +1,148 @@ +// Package sourcespec defines the offline grammar and stable identity of direct +// package sources. It never contacts a host or reads package content. +package sourcespec + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "net/url" + "path" + "regexp" + "strings" + "unicode" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/adapters/pathpolicy" +) + +type Kind string + +const ( + KindGit Kind = "git" + KindArchive Kind = "archive" + KindHTTPSAuto Kind = "https-auto" + KindLocalArchive Kind = "local-archive" +) + +// Git describes intent. Browser is transient and must be resolved before any +// coordinates are persisted. Repository identifies canonical HTTPS GitHub only. +type Git struct { + URL, Repository, Path, Ref, Commit string + Browser *BrowserTree +} + +type BrowserTree struct{ Provider, Suffix string } + +// Archive.URL is a canonical HTTPS endpoint or, for an explicit local input, +// the file path. Local archives are never portable saved requirements. +type Archive struct{ URL, Path, SHA256 string } + +type Spec struct { + Kind Kind + Git *Git + Archive *Archive + Requested string +} + +var ( + commitPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) + digestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + repositoryPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) +) + +func ValidCommit(value string) bool { return commitPattern.MatchString(value) } +func ValidSHA256(value string) bool { return digestPattern.MatchString(value) } + +func safeText(value string) bool { + return value != "" && len(value) <= 8192 && strings.TrimSpace(value) == value && + !strings.ContainsAny(value, `\`) && !strings.ContainsFunc(value, unicode.IsControl) +} + +func NormalizeSubpath(value string) (string, error) { + if value == "" { + return "", nil + } + if len(value) > 1024 || path.IsAbs(value) || path.Clean(value) != value || len(strings.Split(value, "/")) > 64 { + return "", errors.New("package path must be a normalized portable relative directory") + } + for _, segment := range strings.Split(value, "/") { + if err := pathpolicy.ValidatePortablePathSegment(segment); err != nil { + return "", errors.New("package path must be a normalized portable relative directory") + } + } + return value, nil +} + +func ValidateRef(value string) error { + if value == "" || len(value) > 1024 || !safeText(value) || strings.HasPrefix(value, "-") || strings.ContainsAny(value, " ~^:?*[\\") || strings.Contains(value, "..") || strings.Contains(value, "@{") || strings.Contains(value, "//") || strings.HasSuffix(value, ".") || strings.HasSuffix(value, "/") || strings.HasPrefix(value, "/") { + return errors.New("Git selector must be an exact branch, tag or full lowercase commit SHA") + } + for _, segment := range strings.Split(value, "/") { + if strings.HasPrefix(segment, ".") || strings.HasSuffix(segment, ".lock") { + return errors.New("unsafe Git ref") + } + } + return nil +} + +// Key hashes a length-unambiguous canonical tuple; revision is intentionally +// excluded so updates retain ownership. Historical GitHub keys stay unchanged. +func Key(kind Kind, endpoint, packagePath string) string { + body, _ := json.Marshal([3]string{string(kind), endpoint, packagePath}) + sum := sha256.Sum256(body) + prefix := "git-url-v1:" + if kind == KindArchive { + prefix = "archive-url-v1:" + } + return prefix + hex.EncodeToString(sum[:]) +} + +func HasArchiveSuffix(value string) bool { + value = strings.ToLower(value) + for _, suffix := range []string{".zip", ".tar", ".tar.gz", ".tgz"} { + if strings.HasSuffix(value, suffix) { + return true + } + } + return false +} + +func NormalizeArchiveURL(value string) (string, error) { + u, err := normalizedURL(value, "https") + if err != nil || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { + return "", errors.New("archive source requires a credential-free HTTPS URL without query or fragment") + } + return u.String(), nil +} + +func normalizedURL(value, scheme string) (*url.URL, error) { + if !safeText(value) { + return nil, errors.New("unsafe source URL") + } + u, err := url.Parse(value) + if err != nil || u.Scheme != scheme || u.Hostname() == "" || u.Opaque != "" || u.Host != strings.TrimSpace(u.Host) || u.Path == "" || strings.HasPrefix(u.Path, "//") || strings.ContainsAny(u.Path, "\\\x00\r\n\t") { + return nil, errors.New("invalid source URL") + } + if u.Port() != "" { + for _, c := range u.Port() { + if c < '0' || c > '9' { + return nil, errors.New("invalid source port") + } + } + } + // Re-encoding must not silently change endpoint path semantics. + for _, segment := range strings.Split(u.Path, "/") { + if segment == "." || segment == ".." { + return nil, errors.New("unsafe source URL path") + } + } + u.Scheme = strings.ToLower(u.Scheme) + u.Host = strings.ToLower(u.Host) + u.Path = strings.TrimSuffix(u.Path, "/") + if u.Path == "" { + return nil, errors.New("source URL requires a repository or package path") + } + u.RawPath = "" + return u, nil +} diff --git a/install/integrationctl/agentplugins/sourcespec/source_test.go b/install/integrationctl/agentplugins/sourcespec/source_test.go new file mode 100644 index 000000000..543e0b23c --- /dev/null +++ b/install/integrationctl/agentplugins/sourcespec/source_test.go @@ -0,0 +1,78 @@ +package sourcespec + +import ( + "reflect" + "strings" + "testing" +) + +func TestParseDirectSources(t *testing.T) { + sha := strings.Repeat("a", 40) + tests := []struct { + raw string + kind Kind + git *Git + archive *Archive + }{ + {"Owner/Repo@" + sha + "//plugins/demo", KindGit, &Git{URL: "https://github.com/owner/repo.git", Repository: "owner/repo", Commit: sha, Path: "plugins/demo"}, nil}, + {"github:owner/repo@refs/heads/main//pkg", KindGit, &Git{URL: "https://github.com/owner/repo.git", Repository: "owner/repo", Ref: "refs/heads/main", Path: "pkg"}, nil}, + {"https://example.test/Team/Repo.git#v1//pkg", KindGit, &Git{URL: "https://example.test/Team/Repo.git", Ref: "v1", Path: "pkg"}, nil}, + {"git+https://example.test/Team/Repo", KindGit, &Git{URL: "https://example.test/Team/Repo"}, nil}, + {"ssh://git@example.test:2222/Team/Repo#main", KindGit, &Git{URL: "ssh://git@example.test:2222/Team/Repo", Ref: "main"}, nil}, + {"git@ssh.dev.azure.com:v3/org/project/repo", KindGit, &Git{URL: "git@ssh.dev.azure.com:v3/org/project/repo"}, nil}, + {"https://github.com/owner/repo/tree/feature/a/pkg", KindGit, &Git{URL: "https://github.com/owner/repo.git", Repository: "owner/repo", Browser: &BrowserTree{Provider: "github", Suffix: "feature/a/pkg"}}, nil}, + {"https://gitlab.com/group/sub/repo/-/tree/main/pkg", KindGit, &Git{URL: "https://gitlab.com/group/sub/repo.git", Browser: &BrowserTree{Provider: "gitlab", Suffix: "main/pkg"}}, nil}, + {"https://self.test/group/sub/repo/-/tree/main/pkg", KindGit, &Git{URL: "https://self.test/group/sub/repo.git", Browser: &BrowserTree{Provider: "gitlab", Suffix: "main/pkg"}}, nil}, + {"https://org@dev.azure.com/org/project/_git/repo?version=GBfeature%2Fa&path=%2Fplugins%2Fdemo&_a=contents", KindGit, &Git{URL: "https://dev.azure.com/org/project/_git/repo", Ref: "refs/heads/feature/a", Path: "plugins/demo"}, nil}, + {"https://org.visualstudio.com/project/_git/repo?version=GC" + sha, KindGit, &Git{URL: "https://org.visualstudio.com/project/_git/repo", Commit: sha}, nil}, + {"https://github.com/owner/repo/releases/download/v1/plugin.zip", KindArchive, nil, &Archive{URL: "https://github.com/owner/repo/releases/download/v1/plugin.zip"}}, + {"https://gitlab.com/g/r/-/archive/main/r.tar.gz", KindArchive, nil, &Archive{URL: "https://gitlab.com/g/r/-/archive/main/r.tar.gz"}}, + {"https://downloads.test/package#sha256=" + strings.Repeat("b", 64) + "&path=plugins%2Fdemo", KindArchive, nil, &Archive{URL: "https://downloads.test/package", SHA256: strings.Repeat("b", 64), Path: "plugins/demo"}}, + {"https://example.test/package", KindHTTPSAuto, nil, &Archive{URL: "https://example.test/package"}}, + {"./plugin.tgz", KindLocalArchive, nil, &Archive{URL: "./plugin.tgz"}}, + } + for _, tt := range tests { + t.Run(tt.raw, func(t *testing.T) { + got, err := Parse(tt.raw) + if err != nil { + t.Fatal(err) + } + if got.Kind != tt.kind || !reflect.DeepEqual(got.Git, tt.git) || !reflect.DeepEqual(got.Archive, tt.archive) { + t.Fatalf("got %#v git=%#v archive=%#v", got, got.Git, got.Archive) + } + }) + } +} + +func TestUnsafeSourcesFailWithoutEchoingInput(t *testing.T) { + for _, raw := range []string{ + "https://secret:TOKEN@example.test/r.git", "ssh://secret:TOKEN@example.test/r.git", + "https://example.test/a.zip?token=TOKEN", "https://example.test/r.git#--upload-pack=TOKEN", + "https://example.test/r.git#main//../escape", "git+file:///tmp/repo", + "https://github.com/o/r/blob/main/SKILL.md", "https://gitlab.com/g/r/-/blob/main/SKILL.md", + "https://dev.azure.com/o/p/_git/r?token=TOKEN", "https://dev.azure.com/o/p/_git/r?version=GBa&version=GBb", + "https://example.test/r.git#main\nTOKEN", "https://example.test/../r.git", "ssh://git@example.test/repo#main//CON", + } { + t.Run(raw, func(t *testing.T) { + _, err := Parse(raw) + if err == nil { + t.Fatal("unsafe input accepted") + } + if strings.Contains(err.Error(), "TOKEN") { + t.Fatal("credential material leaked") + } + }) + } +} + +func TestStableTupleKeysAreUnambiguousAndCaseSensitive(t *testing.T) { + a := Key(KindGit, "ssh://git@example.test/a:b", "c") + b := Key(KindGit, "ssh://git@example.test/a", "b:c") + c := Key(KindGit, "ssh://git@example.test/a:b", "C") + if a == b || a == c || a == Key(KindArchive, "ssh://git@example.test/a:b", "c") { + t.Fatal("distinct source tuples collided") + } + if !strings.HasPrefix(a, "git-url-v1:") || len(a) != len("git-url-v1:")+64 { + t.Fatalf("invalid key %q", a) + } +} From ce9b26c7c0a39a65b2309d8d60dfdb2b3008c630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Fri, 23 Oct 2026 18:35:04 +0300 Subject: [PATCH 03/25] feat(sources): persist URL Git and checksum-bound archive dependency pins --- .../agentpluginscli/environment_session.go | 6 +- .../agentpluginscli/manifestedit/edit.go | 4 +- .../adapters/dependencymanifest/decode.go | 49 ++++++- .../dependencymanifest/source_test.go | 36 ++++++ .../agentplugins/environment/coordinator.go | 11 +- .../agentplugins/environment/graph.go | 15 ++- .../agentplugins/environment/model.go | 54 ++++++-- .../environment/resolve_expand.go | 9 +- .../agentplugins/environment/resolve_graph.go | 9 +- .../agentplugins/environment/sources.go | 106 +++++++++++++++ .../agentplugins/environment/sources_test.go | 121 ++++++++++++++++++ .../agentplugins/sourcespec/git.go | 8 +- .../agentplugins/sourcespec/parse.go | 7 +- .../agentplugins/sourcespec/provider.go | 8 ++ .../agentplugins/sourcespec/source.go | 4 +- .../agentplugins/sourcespec/source_test.go | 5 + 16 files changed, 419 insertions(+), 33 deletions(-) create mode 100644 install/integrationctl/agentplugins/adapters/dependencymanifest/source_test.go create mode 100644 install/integrationctl/agentplugins/environment/sources.go create mode 100644 install/integrationctl/agentplugins/environment/sources_test.go diff --git a/cli/internal/agentpluginscli/environment_session.go b/cli/internal/agentpluginscli/environment_session.go index 4d8e213bf..0fb2f58bb 100644 --- a/cli/internal/agentpluginscli/environment_session.go +++ b/cli/internal/agentpluginscli/environment_session.go @@ -235,7 +235,7 @@ func dependencyIntentEdits(previous, wanted environment.Requirements) []manifest if ok { v := &manifestedit.Dependency{} if value.Git != nil { - v.Git = "https://github.com/" + value.Git.Repository + ".git" + v.Git = value.Git.Endpoint() v.Path = value.Git.Path switch value.Git.Selector.Kind { case environment.SelectorCommit: @@ -243,6 +243,10 @@ func dependencyIntentEdits(previous, wanted environment.Requirements) []manifest case environment.SelectorRef: v.Ref = value.Git.Selector.Value } + } else if value.Archive != nil { + v.Archive = value.Archive.URL + v.SHA256 = value.Archive.SHA256 + v.Path = value.Archive.Path } else { v.Registry = "uap" v.Distribution = value.Registry.Distribution diff --git a/cli/internal/agentpluginscli/manifestedit/edit.go b/cli/internal/agentpluginscli/manifestedit/edit.go index badb6165a..3b4b388ab 100644 --- a/cli/internal/agentpluginscli/manifestedit/edit.go +++ b/cli/internal/agentpluginscli/manifestedit/edit.go @@ -21,7 +21,7 @@ var ErrUnsupportedLayout = errors.New("unsupported TOML layout; edit uap.toml ma // Empty fields are omitted. Release is an exact sequence represented as a TOML // string; semantic acceptance remains the strict manifest reader's job. type Dependency struct { - Git, Path, Ref, Commit, Registry, Distribution, Release, Version string + Git, Path, Ref, Commit, Registry, Distribution, Release, Version, Archive, SHA256 string } type DependencyEdit struct { @@ -116,7 +116,7 @@ func editDependency(doc *tomledit.Document, edit DependencyEdit) error { section = appendSection(doc, name) } fields := []struct{ name, value string }{ - {"git", edit.Value.Git}, {"path", edit.Value.Path}, {"ref", edit.Value.Ref}, + {"git", edit.Value.Git}, {"archive", edit.Value.Archive}, {"sha256", edit.Value.SHA256}, {"path", edit.Value.Path}, {"ref", edit.Value.Ref}, {"commit", edit.Value.Commit}, {"registry", edit.Value.Registry}, {"distribution", edit.Value.Distribution}, {"release", edit.Value.Release}, } diff --git a/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go b/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go index 6d21cf288..597beb15a 100644 --- a/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go +++ b/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go @@ -11,9 +11,12 @@ import ( "github.com/pelletier/go-toml/v2" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" ) type rawDependency struct { + Archive string `toml:"archive"` + SHA256 string `toml:"sha256"` Version string `toml:"-"` Git string `toml:"git"` Path string `toml:"path"` @@ -155,6 +158,10 @@ func (doc Document) clone() environment.Requirements { source := *value.Git value.Git = &source } + if value.Archive != nil { + source := *value.Archive + value.Archive = &source + } if value.Registry != nil { source := *value.Registry value.Registry = &source @@ -194,14 +201,45 @@ func requirementFromRaw(alias string, raw rawDependency) (environment.Requiremen return environment.Requirement{}, err } gitSource, registrySource := raw.Git != "", raw.Registry != "" || raw.Distribution != "" || raw.Release != nil - if gitSource == registrySource { - return environment.Requirement{}, errors.New("dependency must contain exactly one git or registry source") + count := 0 + for _, yes := range []bool{gitSource, registrySource, raw.Archive != ""} { + if yes { + count++ + } + } + if count != 1 { + return environment.Requirement{}, errors.New("dependency must contain exactly one git, archive or registry source") + } + if raw.Archive != "" { + if raw.Ref != "" || raw.Commit != "" { + return environment.Requirement{}, errors.New("archive source cannot contain Git selectors") + } + endpoint, err := sourcespec.NormalizeArchiveURL(raw.Archive) + if err != nil { + return environment.Requirement{}, err + } + return environment.Requirement{Alias: alias, Archive: &environment.ArchiveSource{URL: endpoint, Path: raw.Path, SHA256: raw.SHA256}}, nil + } + if raw.SHA256 != "" { + return environment.Requirement{}, errors.New("sha256 is only valid for archive sources") } if gitSource { if raw.Registry != "" || raw.Distribution != "" || raw.Release != nil { return environment.Requirement{}, errors.New("git source cannot contain registry fields") } - return environment.Requirement{Alias: alias, Git: &environment.GitSource{Repository: strings.ToLower(strings.TrimSuffix(strings.TrimPrefix(raw.Git, "https://github.com/"), ".git")), Path: raw.Path, Selector: selector}}, nil + endpoint := raw.Git + if !strings.Contains(endpoint, ":") { + endpoint = "https://github.com/" + endpoint + ".git" + } + normalized, repository, err := sourcespec.NormalizeGitURL(endpoint) + if err != nil { + return environment.Requirement{}, err + } + source := &environment.GitSource{Repository: repository, Path: raw.Path, Selector: selector} + if repository == "" { + source.URL = normalized + } + return environment.Requirement{Alias: alias, Git: source}, nil } if raw.Ref != "" || raw.Commit != "" { return environment.Requirement{}, errors.New("registry source cannot contain git selectors") @@ -223,6 +261,9 @@ func rawSelector(raw rawDependency) (environment.Selector, error) { } return environment.Selector{Kind: kind, Value: raw.Version}, nil } + if raw.Archive != "" { + return environment.Selector{Kind: environment.SelectorArchive, Value: raw.SHA256}, nil + } if raw.Git != "" { switch { case raw.Commit != "" && raw.Ref != "": @@ -280,7 +321,7 @@ func validateDependencyFields(path string, body []byte, alias string, value any) } for _, key := range sortedKeys(fields) { switch key { - case "git", "path", "ref", "commit", "registry", "distribution": + case "git", "path", "ref", "commit", "registry", "distribution", "archive", "sha256": if _, ok := fields[key].(string); !ok { return manifestError(path, body, fmt.Errorf("dependencies.%s.%s must be a string", alias, key), "dependencies", alias, key) } diff --git a/install/integrationctl/agentplugins/adapters/dependencymanifest/source_test.go b/install/integrationctl/agentplugins/adapters/dependencymanifest/source_test.go new file mode 100644 index 000000000..43a3fe272 --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/dependencymanifest/source_test.go @@ -0,0 +1,36 @@ +package dependencymanifest + +import ( + "strings" + "testing" +) + +func TestGenericGitAndArchiveDeclarations(t *testing.T) { + body := []byte("schema_version = 1\n[dependencies.git]\ngit = 'ssh://git@example.test/Team/Repo'\nref = 'main'\npath = 'plugins/demo'\n[dependencies.assets]\narchive = 'https://example.test/package'\nsha256 = '" + strings.Repeat("a", 64) + "'\npath = 'plugins/demo'\n[targets]\ncodex = ['git','assets']\n") + doc, err := Decode(body, "uap.toml") + if err != nil { + t.Fatal(err) + } + reqs, err := doc.Root() + if err != nil { + t.Fatal(err) + } + if reqs.Dependencies["git"].Git.URL != "ssh://git@example.test/Team/Repo" || reqs.Dependencies["assets"].Archive.SHA256 != strings.Repeat("a", 64) { + t.Fatalf("coordinates lost: %+v", reqs) + } + for _, change := range []string{ + "git = 'https://user:TOKEN@example.test/repo.git'\nref='main'", + "archive = 'https://example.test/pkg.zip'\nref='main'", + "archive = 'https://example.test/pkg.zip'\ngit='acme/repo'", + "git='acme/repo'\nsha256='" + strings.Repeat("a", 64) + "'", + "archive='https://example.test/pkg.zip?token=TOKEN'", + } { + _, err := Decode([]byte("schema_version=1\n[dependencies.demo]\n"+change+"\n[targets]\ncodex=['demo']\n"), "uap.toml") + if err == nil { + t.Fatalf("invalid mixed source accepted: %s", change) + } + if strings.Contains(err.Error(), "TOKEN") { + t.Fatal("URL credentials leaked") + } + } +} diff --git a/install/integrationctl/agentplugins/environment/coordinator.go b/install/integrationctl/agentplugins/environment/coordinator.go index 83d688439..5d119ab3c 100644 --- a/install/integrationctl/agentplugins/environment/coordinator.go +++ b/install/integrationctl/agentplugins/environment/coordinator.go @@ -46,6 +46,7 @@ func (coordinator Coordinator) Resolve(ctx context.Context, requirements Require } moving := selected.Git != nil && selected.Git.Selector.Kind != SelectorCommit moving = moving || selected.Registry != nil && selected.Registry.Selector.Kind == SelectorRange + moving = moving || selected.Archive != nil && selected.Archive.SHA256 == "" if !moving { return Lock{}, errors.New("exact commit/release/version pins must be changed in the manifest") } @@ -159,12 +160,7 @@ func syncAction(pkg LockedPackage, observed BindingObservation) (DesiredAction, if observed.SourceKey != "" && observed.SourceKey != pkg.SourceKey { return "", "source_change" } - revision := "" - if pkg.Source.Git != nil { - revision = pkg.Source.Git.Commit - } else if pkg.Source.Registry != nil { - revision = pkg.Source.Registry.Commit - } + revision := lockedRevision(pkg) if observed.PackageDigest == pkg.TreeDigest && observed.Revision == revision && !observed.Incomplete { return ActionUnchanged, "pinned package digest already matches" } @@ -248,6 +244,9 @@ func sortSyncBindings(bindings []PreparedBinding) { } func lockedRevision(pkg LockedPackage) string { + if pkg.Source.Archive != nil { + return "sha256:" + pkg.Source.Archive.SHA256 + } if pkg.Source.Git != nil { return pkg.Source.Git.Commit } diff --git a/install/integrationctl/agentplugins/environment/graph.go b/install/integrationctl/agentplugins/environment/graph.go index d8682aca6..c578890ec 100644 --- a/install/integrationctl/agentplugins/environment/graph.go +++ b/install/integrationctl/agentplugins/environment/graph.go @@ -50,6 +50,11 @@ func pinSourceIdentity(source LockedSource) LockedSource { cloned.Requested = Selector{} source.Registry = &cloned } + if source.Archive != nil { + cloned := *source.Archive + cloned.RequestedSHA256 = "" + source.Archive = &cloned + } return source } @@ -102,12 +107,16 @@ func validatePin(pkg LockedPackage) error { req := Requirement{Alias: pkg.Alias} if pkg.Source.Git != nil { source := pkg.Source.Git - req.Git = &GitSource{Repository: source.Repository, Path: source.Path, Selector: source.Requested} + req.Git = &GitSource{Repository: source.Repository, URL: source.URL, Path: source.Path, Selector: source.Requested} } if pkg.Source.Registry != nil { source := pkg.Source.Registry req.Registry = &RegistrySource{Distribution: source.Distribution, Selector: source.Requested} } + if pkg.Source.Archive != nil { + source := pkg.Source.Archive + req.Archive = &ArchiveSource{URL: source.URL, Path: source.Path, SHA256: source.RequestedSHA256} + } key, err := req.SourceKey() if err != nil { return err @@ -154,6 +163,10 @@ func validateEdges(packages map[string]LockedPackage) error { if edge.Selector != pkg.Source.Git.Requested { return fmt.Errorf("%s: Git selector differs from consumer %s", pkg.Alias, edge.Alias) } + } else if pkg.Source.Archive != nil { + if edge.Selector != (Selector{Kind: SelectorArchive, Value: pkg.Source.Archive.RequestedSHA256}) { + return fmt.Errorf("%s: archive selector differs from consumer %s", pkg.Alias, edge.Alias) + } } else if edge.Selector.Kind != SelectorRelease && edge.Selector.Kind != SelectorVersion && edge.Selector.Kind != SelectorRange { return fmt.Errorf("%s: invalid registry dependency selector", pkg.Alias) } else if !registrySelectorMatches(edge.Selector, pkg) { diff --git a/install/integrationctl/agentplugins/environment/model.go b/install/integrationctl/agentplugins/environment/model.go index e72403f31..b7d9caafb 100644 --- a/install/integrationctl/agentplugins/environment/model.go +++ b/install/integrationctl/agentplugins/environment/model.go @@ -15,6 +15,7 @@ import ( "strings" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" ) const ( @@ -34,6 +35,7 @@ var ( type SelectorKind string const ( + SelectorArchive SelectorKind = "archive" SelectorCommit SelectorKind = "commit" SelectorRef SelectorKind = "ref" SelectorDefault SelectorKind = "default" @@ -48,7 +50,8 @@ type Selector struct { } type GitSource struct { - Repository string `json:"repository"` + URL string `json:"url,omitempty"` + Repository string `json:"repository,omitempty"` Path string `json:"path,omitempty"` Selector Selector `json:"selector"` } @@ -58,7 +61,21 @@ type RegistrySource struct { Selector Selector `json:"selector"` } +type ArchiveSource struct { + URL string `json:"url"` + Path string `json:"path,omitempty"` + SHA256 string `json:"sha256,omitempty"` +} + +type LockedArchiveSource struct { + URL string `json:"url"` + Path string `json:"path,omitempty"` + RequestedSHA256 string `json:"requested_sha256,omitempty"` + SHA256 string `json:"sha256"` +} + type Requirement struct { + Archive *ArchiveSource `json:"archive,omitempty"` Alias string `json:"alias"` Git *GitSource `json:"git,omitempty"` Registry *RegistrySource `json:"registry,omitempty"` @@ -71,12 +88,14 @@ type Requirements struct { } type LockedSource struct { + Archive *LockedArchiveSource `json:"archive,omitempty"` Git *LockedGitSource `json:"git,omitempty"` Registry *LockedRegistrySource `json:"registry,omitempty"` } type LockedGitSource struct { - Repository string `json:"repository"` + URL string `json:"url,omitempty"` + Repository string `json:"repository,omitempty"` Path string `json:"path,omitempty"` Requested Selector `json:"requested"` Commit string `json:"commit"` @@ -159,6 +178,10 @@ type PreparedSync struct { func (selector Selector) Validate() error { switch selector.Kind { + case SelectorArchive: + if selector.Value != "" && !sourcespec.ValidSHA256(selector.Value) { + return errors.New("archive selector requires a lowercase SHA256 or a moving URL") + } case SelectorCommit: if !commitPattern.MatchString(selector.Value) { return errors.New("commit selector requires one full lowercase 40-character SHA") @@ -212,12 +235,15 @@ func (requirement Requirement) Validate() error { if !validDeclarationKey(requirement.Alias) { return fmt.Errorf("alias %q must match %s", requirement.Alias, aliasPattern) } - if (requirement.Git == nil) == (requirement.Registry == nil) { - return errors.New("dependency must contain exactly one git or registry source") + if sourceCount(requirement.Git != nil, requirement.Registry != nil, requirement.Archive != nil) != 1 { + return errors.New("dependency must contain exactly one git, archive or registry source") + } + if requirement.Archive != nil { + return requirement.Archive.Validate() } if requirement.Git != nil { - if !repositoryPattern.MatchString(requirement.Git.Repository) { - return errors.New("git repository must be owner/repo") + if err := requirement.Git.ValidateEndpoint(); err != nil { + return err } if err := validateSubpath(requirement.Git.Path); err != nil { return err @@ -296,8 +322,14 @@ func (requirement Requirement) SourceKey() (string, error) { return "", err } if requirement.Git != nil { + if requirement.Git.URL != "" { + return sourcespec.Key(sourcespec.KindGit, requirement.Git.URL, requirement.Git.Path), nil + } return "github:" + strings.ToLower(requirement.Git.Repository) + ":" + requirement.Git.Path, nil } + if requirement.Archive != nil { + return sourcespec.Key(sourcespec.KindArchive, requirement.Archive.URL, requirement.Archive.Path), nil + } return "directory:" + strings.ToLower(requirement.Registry.Distribution), nil } @@ -334,6 +366,9 @@ func ManifestDigest(requirements Requirements) (string, error) { if requirement.Git != nil { item.Path = requirement.Git.Path item.Selector = requirement.Git.Selector + } else if requirement.Archive != nil { + item.Path = requirement.Archive.Path + item.Selector = requirement.RequestedSelector() } else { item.Selector = requirement.Registry.Selector } @@ -390,7 +425,7 @@ func (lock Lock) ValidateCompatiblePins(_ Requirements) error { func validateLockedSelector(requirement Requirement, pkg LockedPackage) error { if requirement.Git != nil { - if pkg.Source.Git == nil || pkg.Source.Registry != nil || !strings.EqualFold(pkg.Source.Git.Repository, requirement.Git.Repository) || + if pkg.Source.Git == nil || pkg.Source.Registry != nil || pkg.Source.Archive != nil || pkg.Source.Git.URL != requirement.Git.URL || !strings.EqualFold(pkg.Source.Git.Repository, requirement.Git.Repository) || pkg.Source.Git.Path != requirement.Git.Path || pkg.Source.Git.Requested != requirement.Git.Selector || !commitPattern.MatchString(pkg.Source.Git.Commit) { return errors.New("locked Git identity does not match manifest") @@ -400,12 +435,15 @@ func validateLockedSelector(requirement Requirement, pkg LockedPackage) error { } return nil } + if requirement.Archive != nil { + return validateLockedArchive(requirement, pkg) + } return validateLockedRegistrySelector(requirement, pkg) } func validateLockedRegistrySelector(requirement Requirement, pkg LockedPackage) error { source := pkg.Source.Registry - if source == nil || pkg.Source.Git != nil || !strings.EqualFold(source.Distribution, requirement.Registry.Distribution) || + if source == nil || pkg.Source.Git != nil || pkg.Source.Archive != nil || !strings.EqualFold(source.Distribution, requirement.Registry.Distribution) || source.Requested != requirement.Registry.Selector || source.ReleaseSequence == 0 || !registrySelectorMatches(requirement.Registry.Selector, pkg) || !strings.EqualFold(source.DistributionID, source.Distribution) || !distributionPattern.MatchString(source.DistributionID) || !repositoryPattern.MatchString(source.Repository) || validateSubpath(source.Path) != nil || !commitPattern.MatchString(source.Commit) { return errors.New("locked registry identity does not match manifest") diff --git a/install/integrationctl/agentplugins/environment/resolve_expand.go b/install/integrationctl/agentplugins/environment/resolve_expand.go index ddbfbb486..476aa9ebf 100644 --- a/install/integrationctl/agentplugins/environment/resolve_expand.go +++ b/install/integrationctl/agentplugins/environment/resolve_expand.go @@ -80,10 +80,7 @@ func validateGraphAliases(nodes map[string]graphNode) error { } func requirementSelector(req Requirement) Selector { - if req.Git != nil { - return req.Git.Selector - } - return req.Registry.Selector + return req.RequestedSelector() } func (solver *graphSolver) nodeRequirement(key string, req Requirement, choices map[string]ResolvedPackage) Requirement { @@ -98,8 +95,10 @@ func (solver *graphSolver) nodeRequirement(key string, req Requirement, choices // package-local declaration names do not name environment bindings. if req.Registry != nil { req.Alias = req.Registry.Distribution - } else { + } else if req.Git != nil && req.Git.Repository != "" { req.Alias = req.Git.Repository + } else { + req.Alias = "candidate" } } return req diff --git a/install/integrationctl/agentplugins/environment/resolve_graph.go b/install/integrationctl/agentplugins/environment/resolve_graph.go index afd381018..459d81e26 100644 --- a/install/integrationctl/agentplugins/environment/resolve_graph.go +++ b/install/integrationctl/agentplugins/environment/resolve_graph.go @@ -125,6 +125,11 @@ func setRequested(pkg *LockedPackage, req Requirement) { cloned.Requested = req.Git.Selector pkg.Source.Git = &cloned } + if pkg.Source.Archive != nil { + cloned := *pkg.Source.Archive + cloned.RequestedSHA256 = req.Archive.SHA256 + pkg.Source.Archive = &cloned + } } func (solver *graphSolver) search(ctx context.Context, choices map[string]ResolvedPackage) (map[string]ResolvedPackage, map[string]graphNode, error) { @@ -188,7 +193,7 @@ func (solver *graphSolver) choose(ctx context.Context, key string, node graphNod } } } - if node.requirement.Git != nil || node.requirement.Registry.Selector.Kind == SelectorRelease { + if node.requirement.Git != nil || node.requirement.Archive != nil || node.requirement.Registry.Selector.Kind == SelectorRelease { return solver.chooseExact(ctx, key, node, choices, last) } return solver.chooseCandidates(ctx, key, node, choices, last) @@ -303,7 +308,7 @@ func conflictFor(key string, node graphNode) error { if demand.requirement.Registry != nil { selector = requirementSelector(demand.requirement) } else { - selector = demand.requirement.Git.Selector + selector = requirementSelector(demand.requirement) } parts = append(parts, name+" "+string(selector.Kind)+" "+selector.Value) } diff --git a/install/integrationctl/agentplugins/environment/sources.go b/install/integrationctl/agentplugins/environment/sources.go new file mode 100644 index 000000000..52d79fd1c --- /dev/null +++ b/install/integrationctl/agentplugins/environment/sources.go @@ -0,0 +1,106 @@ +package environment + +import ( + "errors" + "strings" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +func sourceCount(values ...bool) int { + n := 0 + for _, value := range values { + if value { + n++ + } + } + return n +} + +func (source GitSource) ValidateEndpoint() error { + if (source.Repository == "") == (source.URL == "") { + return errors.New("git source requires exactly one repository or URL") + } + if source.URL == "" { + if !repositoryPattern.MatchString(source.Repository) { + return errors.New("git repository must be owner/repo") + } + return nil + } + endpoint, repository, err := sourcespec.NormalizeGitURL(source.URL) + if err != nil || endpoint != source.URL || repository != "" { + return errors.New("git URL must be a normalized HTTPS/SSH endpoint; use repository for canonical GitHub HTTPS") + } + return nil +} + +func (source GitSource) Endpoint() string { + if source.URL != "" { + return source.URL + } + return "https://github.com/" + source.Repository + ".git" +} + +func (source ArchiveSource) Validate() error { + endpoint, err := sourcespec.NormalizeArchiveURL(source.URL) + if err != nil || endpoint != source.URL { + return errors.New("archive dependency requires a normalized credential-free HTTPS URL") + } + if _, err := sourcespec.NormalizeSubpath(source.Path); err != nil { + return err + } + if source.SHA256 != "" && !sourcespec.ValidSHA256(source.SHA256) { + return errors.New("archive sha256 requires 64 lowercase hexadecimal characters") + } + return nil +} + +func (requirement Requirement) RequestedSelector() Selector { + if requirement.Git != nil { + return requirement.Git.Selector + } + if requirement.Archive != nil { + return Selector{Kind: SelectorArchive, Value: requirement.Archive.SHA256} + } + if requirement.Registry != nil { + return requirement.Registry.Selector + } + return Selector{} +} + +func validateLockedArchive(requirement Requirement, pkg LockedPackage) error { + a := pkg.Source.Archive + r := requirement.Archive + if a == nil || pkg.Source.Git != nil || pkg.Source.Registry != nil || a.URL != r.URL || a.Path != r.Path || a.RequestedSHA256 != r.SHA256 || !sourcespec.ValidSHA256(a.SHA256) || (r.SHA256 != "" && r.SHA256 != a.SHA256) { + return errors.New("locked archive identity does not match manifest") + } + return nil +} + +// Requirement reconstructs only portable intent from a validated pin. It never +// adopts mutable store provenance or a machine's profile/project paths. +func (pkg LockedPackage) Requirement() Requirement { + r := Requirement{Alias: pkg.Alias} + if s := pkg.Source.Git; s != nil { + r.Git = &GitSource{Repository: s.Repository, URL: s.URL, Path: s.Path, Selector: s.Requested} + } + if s := pkg.Source.Archive; s != nil { + r.Archive = &ArchiveSource{URL: s.URL, Path: s.Path, SHA256: s.RequestedSHA256} + } + if s := pkg.Source.Registry; s != nil { + r.Registry = &RegistrySource{Distribution: s.Distribution, Selector: s.Requested} + } + return r +} + +// Revision returns the immutable version fact used by all lifecycle receipts. +func (pkg LockedPackage) Revision() string { return lockedRevision(pkg) } + +func (requirement Requirement) SameSource(other Requirement) bool { + a, err := requirement.SourceKey() + if err != nil { + return false + } + b, err := other.SourceKey() + return err == nil && strings.Compare(a, b) == 0 +} diff --git a/install/integrationctl/agentplugins/environment/sources_test.go b/install/integrationctl/agentplugins/environment/sources_test.go new file mode 100644 index 000000000..387014493 --- /dev/null +++ b/install/integrationctl/agentplugins/environment/sources_test.go @@ -0,0 +1,121 @@ +package environment + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" +) + +func TestRemoteSourceLockIdentityAndOldReaderRefusal(t *testing.T) { + for _, requirement := range []Requirement{ + {Alias: "demo", Git: &GitSource{URL: "ssh://git@example.test:2222/Team/Repo", Path: "plugins/demo", Selector: Selector{Kind: SelectorRef, Value: "main"}}}, + {Alias: "demo", Archive: &ArchiveSource{URL: "https://example.test/releases/package", Path: "plugins/demo"}}, + } { + reqs := Requirements{SchemaVersion: 1, Dependencies: map[string]Requirement{"demo": requirement}, Targets: map[string][]string{"codex": {"demo"}}} + resolver := remoteGraphResolver{} + lock, err := (Coordinator{Resolver: resolver}).Resolve(t.Context(), reqs, Lock{}, "") + if err != nil { + t.Fatal(err) + } + if err := lock.Validate(reqs); err != nil { + t.Fatal(err) + } + body, err := json.Marshal(lock.Packages[0].Source) + if err != nil { + t.Fatal(err) + } + var old struct { + Git *struct { + Repository, Path string + Requested Selector + Commit string + } `json:"git,omitempty"` + Registry *LockedRegistrySource `json:"registry,omitempty"` + } + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + if decoder.Decode(&old) == nil { + t.Fatal("old reader silently accepted new source semantics") + } + if !reflect.DeepEqual(requirement, lock.Packages[0].Requirement()) { + t.Fatalf("intent changed after lock round trip: %+v", lock) + } + changed := lock.Packages[0] + if changed.Source.Git != nil { + copy := *changed.Source.Git + copy.URL = "ssh://git@other.test/Team/Repo" + changed.Source.Git = © + } else { + copy := *changed.Source.Archive + copy.SHA256 = "bad" + changed.Source.Archive = © + } + lock.Packages[0] = changed + if lock.Validate(reqs) == nil { + t.Fatal("rewritten pin accepted") + } + } +} + +type remoteGraphResolver struct{} + +func (remoteGraphResolver) Resolve(_ context.Context, req Requirement, targets []string, old *LockedPackage, update bool) (LockedPackage, error) { + if old != nil && !update { + return *old, nil + } + key, err := req.SourceKey() + if err != nil { + return LockedPackage{}, err + } + pkg := LockedPackage{Alias: req.Alias, SourceKey: key, DigestAlgorithm: TreeDigestAlgorithm, TreeDigest: "sha256:" + strings.Repeat("a", 64), ManifestDigest: "sha256:" + strings.Repeat("b", 64), Targets: targets} + if g := req.Git; g != nil { + pkg.Source.Git = &LockedGitSource{Repository: g.Repository, URL: g.URL, Path: g.Path, Requested: g.Selector, Commit: strings.Repeat("1", 40)} + } + if a := req.Archive; a != nil { + pkg.Source.Archive = &LockedArchiveSource{URL: a.URL, Path: a.Path, RequestedSHA256: a.SHA256, SHA256: strings.Repeat("2", 64)} + if a.SHA256 != "" { + pkg.Source.Archive.SHA256 = a.SHA256 + } + } + return pkg, nil +} +func (remoteGraphResolver) Candidates(context.Context, Requirement, []string) ([]Candidate, error) { + return nil, errors.New("remote sources must not enumerate Directory versions") +} +func (remoteGraphResolver) AcquireCandidate(context.Context, Requirement, Candidate, []string) (ResolvedPackage, error) { + return ResolvedPackage{}, errors.New("unexpected Directory acquisition") +} +func (remoteGraphResolver) DescribePinned(_ context.Context, pkg LockedPackage) (ResolvedPackage, error) { + return ResolvedPackage{Package: pkg}, nil +} +func (remoteGraphResolver) Eligible(context.Context, LockedPackage, []string) (bool, error) { + return true, nil +} + +func TestArchiveMovingUpdateAndSyncRevision(t *testing.T) { + req := Requirement{Alias: "demo", Archive: &ArchiveSource{URL: "https://example.test/package"}} + reqs := Requirements{SchemaVersion: 1, Dependencies: map[string]Requirement{"demo": req}, Targets: map[string][]string{"codex": {"demo"}}} + coordinator := Coordinator{Resolver: remoteGraphResolver{}} + lock, err := coordinator.Resolve(t.Context(), reqs, Lock{}, "") + if err != nil { + t.Fatal(err) + } + updated, err := coordinator.Resolve(t.Context(), reqs, lock, "demo") + if err != nil { + t.Fatal(err) + } + pkg := updated.Packages[0] + plan, err := coordinator.PrepareSync(reqs, updated, []BindingObservation{{Alias: "demo", ClientID: "codex", Managed: true, SourceKey: pkg.SourceKey, PackageDigest: pkg.TreeDigest, Revision: "sha256:" + pkg.Source.Archive.SHA256}}) + if err != nil || len(plan.Bindings) != 1 || plan.Bindings[0].Action != ActionUnchanged { + t.Fatalf("archive replay changed: %+v %v", plan, err) + } + req.Archive.SHA256 = pkg.Source.Archive.SHA256 + reqs.Dependencies["demo"] = req + if _, err := coordinator.Resolve(t.Context(), reqs, Lock{}, "demo"); err == nil { + t.Fatal("explicit checksum updated without intent change") + } +} diff --git a/install/integrationctl/agentplugins/sourcespec/git.go b/install/integrationctl/agentplugins/sourcespec/git.go index e95554398..11ab71bcf 100644 --- a/install/integrationctl/agentplugins/sourcespec/git.go +++ b/install/integrationctl/agentplugins/sourcespec/git.go @@ -30,7 +30,10 @@ func NormalizeGitURL(value string) (string, string, error) { if strings.ContainsAny(host, "[]: ") || strings.ContainsAny(user, ": ") { return "", "", errors.New("invalid SSH endpoint") } - for _, segment := range strings.Split(right, "/") { + for index, segment := range strings.Split(right, "/") { + if index == 0 && segment == "" && strings.HasPrefix(right, "/") { + continue + } if segment == "." || segment == ".." || segment == "" { return "", "", errors.New("unsafe SSH repository path") } @@ -63,6 +66,9 @@ func NormalizeGitURL(value string) (string, string, error) { } repository := "" if u.Scheme == "https" && u.Host == "github.com" { + if encodedSeparator(u.RawPath) { + return "", "", errors.New("encoded repository separators require an unambiguous clone URL") + } repository = strings.TrimSuffix(strings.TrimPrefix(u.Path, "/"), ".git") if !repositoryPattern.MatchString(repository) { return "", "", errors.New("GitHub repository must be owner/repo") diff --git a/install/integrationctl/agentplugins/sourcespec/parse.go b/install/integrationctl/agentplugins/sourcespec/parse.go index 55c7ca7e5..c304d9186 100644 --- a/install/integrationctl/agentplugins/sourcespec/parse.go +++ b/install/integrationctl/agentplugins/sourcespec/parse.go @@ -24,7 +24,7 @@ func Parse(raw string) (Spec, error) { return out, err } if !strings.HasPrefix(raw, "https://") { - if HasArchiveSuffix(raw) && (strings.HasPrefix(raw, ".") || strings.HasPrefix(raw, "/") || strings.ContainsAny(raw, "/\\")) { + if HasArchiveSuffix(raw) { out.Kind = KindLocalArchive out.Archive = &Archive{URL: raw} return out, nil @@ -40,6 +40,11 @@ func Parse(raw string) (Spec, error) { out.Kind, out.Archive = KindArchive, &a return out, err } + if isAzureHost(strings.ToLower(u.Hostname())) && (u.RawQuery != "" || u.ForceQuery) { + g, _, err := parseProvider(u) + out.Kind, out.Git = KindGit, &g + return out, err + } if u.Fragment != "" || strings.HasSuffix(u.Path, ".git") || strings.Contains(u.Path, ".git//") { g, err := parseGit(raw) out.Kind, out.Git = KindGit, &g diff --git a/install/integrationctl/agentplugins/sourcespec/provider.go b/install/integrationctl/agentplugins/sourcespec/provider.go index ca8a54faa..9dec28789 100644 --- a/install/integrationctl/agentplugins/sourcespec/provider.go +++ b/install/integrationctl/agentplugins/sourcespec/provider.go @@ -8,6 +8,9 @@ import ( func parseProvider(u *url.URL) (Git, bool, error) { host := strings.ToLower(u.Hostname()) + if encodedSeparator(u.RawPath) && (host == "github.com" || host == "gitlab.com" || isAzureHost(host) || strings.Contains(u.Path, "/-/tree/")) { + return Git{}, true, errors.New("encoded browser path separators are ambiguous; use the clone URL with an explicit ref and package path") + } if isAzureHost(host) { g, err := parseAzure(u) return g, true, err @@ -131,3 +134,8 @@ func parseAzure(u *url.URL) (Git, error) { } return g, nil } + +func encodedSeparator(raw string) bool { + value := strings.ToLower(raw) + return strings.Contains(value, "%2f") || strings.Contains(value, "%5c") +} diff --git a/install/integrationctl/agentplugins/sourcespec/source.go b/install/integrationctl/agentplugins/sourcespec/source.go index 3ba9ccb12..b643d4623 100644 --- a/install/integrationctl/agentplugins/sourcespec/source.go +++ b/install/integrationctl/agentplugins/sourcespec/source.go @@ -121,7 +121,7 @@ func normalizedURL(value, scheme string) (*url.URL, error) { return nil, errors.New("unsafe source URL") } u, err := url.Parse(value) - if err != nil || u.Scheme != scheme || u.Hostname() == "" || u.Opaque != "" || u.Host != strings.TrimSpace(u.Host) || u.Path == "" || strings.HasPrefix(u.Path, "//") || strings.ContainsAny(u.Path, "\\\x00\r\n\t") { + if err != nil || u.Scheme != scheme || u.Hostname() == "" || u.Opaque != "" || u.Host != strings.TrimSpace(u.Host) || u.Path == "" || strings.HasPrefix(u.Path, "//") || strings.ContainsAny(u.Path, "\\\x00\r\n\t") || strings.ContainsFunc(u.Path, unicode.IsControl) { return nil, errors.New("invalid source URL") } if u.Port() != "" { @@ -143,6 +143,6 @@ func normalizedURL(value, scheme string) (*url.URL, error) { if u.Path == "" { return nil, errors.New("source URL requires a repository or package path") } - u.RawPath = "" + u.RawPath = strings.TrimSuffix(u.RawPath, "/") return u, nil } diff --git a/install/integrationctl/agentplugins/sourcespec/source_test.go b/install/integrationctl/agentplugins/sourcespec/source_test.go index 543e0b23c..3b4ebfd92 100644 --- a/install/integrationctl/agentplugins/sourcespec/source_test.go +++ b/install/integrationctl/agentplugins/sourcespec/source_test.go @@ -17,19 +17,23 @@ func TestParseDirectSources(t *testing.T) { {"Owner/Repo@" + sha + "//plugins/demo", KindGit, &Git{URL: "https://github.com/owner/repo.git", Repository: "owner/repo", Commit: sha, Path: "plugins/demo"}, nil}, {"github:owner/repo@refs/heads/main//pkg", KindGit, &Git{URL: "https://github.com/owner/repo.git", Repository: "owner/repo", Ref: "refs/heads/main", Path: "pkg"}, nil}, {"https://example.test/Team/Repo.git#v1//pkg", KindGit, &Git{URL: "https://example.test/Team/Repo.git", Ref: "v1", Path: "pkg"}, nil}, + {"https://example.test/Team%2FRepo.git", KindGit, &Git{URL: "https://example.test/Team%2FRepo.git"}, nil}, {"git+https://example.test/Team/Repo", KindGit, &Git{URL: "https://example.test/Team/Repo"}, nil}, {"ssh://git@example.test:2222/Team/Repo#main", KindGit, &Git{URL: "ssh://git@example.test:2222/Team/Repo", Ref: "main"}, nil}, {"git@ssh.dev.azure.com:v3/org/project/repo", KindGit, &Git{URL: "git@ssh.dev.azure.com:v3/org/project/repo"}, nil}, + {"git@example.test:/srv/repo.git", KindGit, &Git{URL: "git@example.test:/srv/repo.git"}, nil}, {"https://github.com/owner/repo/tree/feature/a/pkg", KindGit, &Git{URL: "https://github.com/owner/repo.git", Repository: "owner/repo", Browser: &BrowserTree{Provider: "github", Suffix: "feature/a/pkg"}}, nil}, {"https://gitlab.com/group/sub/repo/-/tree/main/pkg", KindGit, &Git{URL: "https://gitlab.com/group/sub/repo.git", Browser: &BrowserTree{Provider: "gitlab", Suffix: "main/pkg"}}, nil}, {"https://self.test/group/sub/repo/-/tree/main/pkg", KindGit, &Git{URL: "https://self.test/group/sub/repo.git", Browser: &BrowserTree{Provider: "gitlab", Suffix: "main/pkg"}}, nil}, {"https://org@dev.azure.com/org/project/_git/repo?version=GBfeature%2Fa&path=%2Fplugins%2Fdemo&_a=contents", KindGit, &Git{URL: "https://dev.azure.com/org/project/_git/repo", Ref: "refs/heads/feature/a", Path: "plugins/demo"}, nil}, {"https://org.visualstudio.com/project/_git/repo?version=GC" + sha, KindGit, &Git{URL: "https://org.visualstudio.com/project/_git/repo", Commit: sha}, nil}, + {"https://dev.azure.com/org/project/_git/repo.git?version=GBmain", KindGit, &Git{URL: "https://dev.azure.com/org/project/_git/repo.git", Ref: "refs/heads/main"}, nil}, {"https://github.com/owner/repo/releases/download/v1/plugin.zip", KindArchive, nil, &Archive{URL: "https://github.com/owner/repo/releases/download/v1/plugin.zip"}}, {"https://gitlab.com/g/r/-/archive/main/r.tar.gz", KindArchive, nil, &Archive{URL: "https://gitlab.com/g/r/-/archive/main/r.tar.gz"}}, {"https://downloads.test/package#sha256=" + strings.Repeat("b", 64) + "&path=plugins%2Fdemo", KindArchive, nil, &Archive{URL: "https://downloads.test/package", SHA256: strings.Repeat("b", 64), Path: "plugins/demo"}}, {"https://example.test/package", KindHTTPSAuto, nil, &Archive{URL: "https://example.test/package"}}, {"./plugin.tgz", KindLocalArchive, nil, &Archive{URL: "./plugin.tgz"}}, + {"plugin.zip", KindLocalArchive, nil, &Archive{URL: "plugin.zip"}}, } for _, tt := range tests { t.Run(tt.raw, func(t *testing.T) { @@ -52,6 +56,7 @@ func TestUnsafeSourcesFailWithoutEchoingInput(t *testing.T) { "https://github.com/o/r/blob/main/SKILL.md", "https://gitlab.com/g/r/-/blob/main/SKILL.md", "https://dev.azure.com/o/p/_git/r?token=TOKEN", "https://dev.azure.com/o/p/_git/r?version=GBa&version=GBb", "https://example.test/r.git#main\nTOKEN", "https://example.test/../r.git", "ssh://git@example.test/repo#main//CON", + "https://github.com/owner%2Frepo.git", "https://gitlab.com/g/r/-/tree/feature%2Fa/pkg", "https://example.test/r%01.git", } { t.Run(raw, func(t *testing.T) { _, err := Parse(raw) From 5d50808ed60bf7c1eb7695859f24ab761ef63765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Fri, 23 Oct 2026 18:46:13 +0300 Subject: [PATCH 04/25] feat(sources): acquire native HTTPS and SSH Git without executing package filters --- .../adapters/dependencymanifest/decode.go | 3 + .../dependencymanifest/source_test.go | 1 + .../adapters/sourceacquisition/acquirer.go | 48 +++- .../adapters/sourceacquisition/git_direct.go | 261 ++++++++++++++++++ .../sourceacquisition/git_ssh_test.go | 139 ++++++++++ .../sourceacquisition/git_transport.go | 93 +++++++ .../sourceacquisition/git_transport_test.go | 188 +++++++++++++ 7 files changed, 727 insertions(+), 6 deletions(-) create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport_test.go diff --git a/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go b/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go index 597beb15a..9e20dbef6 100644 --- a/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go +++ b/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go @@ -193,6 +193,9 @@ func decodeRawDependency(value any) (rawDependency, error) { func requirementFromRaw(alias string, raw rawDependency) (environment.Requirement, error) { if raw.Version != "" { + if raw.Git != "" || raw.Archive != "" || raw.SHA256 != "" || raw.Path != "" || raw.Ref != "" || raw.Commit != "" || raw.Registry != "" || raw.Distribution != "" || raw.Release != nil { + return environment.Requirement{}, errors.New("version shortcut cannot contain source table fields") + } selector, err := rawSelector(raw) return environment.Requirement{Alias: alias, Registry: &environment.RegistrySource{Distribution: alias, Selector: selector}}, err } diff --git a/install/integrationctl/agentplugins/adapters/dependencymanifest/source_test.go b/install/integrationctl/agentplugins/adapters/dependencymanifest/source_test.go index 43a3fe272..6fbaa8f79 100644 --- a/install/integrationctl/agentplugins/adapters/dependencymanifest/source_test.go +++ b/install/integrationctl/agentplugins/adapters/dependencymanifest/source_test.go @@ -21,6 +21,7 @@ func TestGenericGitAndArchiveDeclarations(t *testing.T) { for _, change := range []string{ "git = 'https://user:TOKEN@example.test/repo.git'\nref='main'", "archive = 'https://example.test/pkg.zip'\nref='main'", + "archive = 'https://example.test/pkg.zip'\nversion='1.2.3'", "archive = 'https://example.test/pkg.zip'\ngit='acme/repo'", "git='acme/repo'\nsha256='" + strings.Repeat("a", 64) + "'", "archive='https://example.test/pkg.zip?token=TOKEN'", diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go index d0cfec707..2439edee7 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go @@ -37,6 +37,8 @@ type Command struct { Args []string Stdin []byte MaxOutputBytes int + // NativeCredentials is restricted to direct user-selected Git transport commands. + NativeCredentials bool } type Runner interface { @@ -49,9 +51,19 @@ func (OSRunner) Run(ctx context.Context, command Command) ([]byte, error) { if len(command.Stdin) != 0 { return nil, fmt.Errorf("git stdin is unsupported by the contained runner") } + environment := isolatedGitEnvironment(os.Environ(), command.Dir) + args := command.Args + if command.NativeCredentials { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + environment = nativeGitEnvironment(os.Environ()) + environment = withBatchSSH(ctx, environment, command.Dir) + args = append(nativeGitOptions(command.Dir), args...) + } result, err := (processadapter.OS{}).RunWithTreeExitGrace(ctx, legacyports.Command{ - Argv: append([]string{"git"}, command.Args...), Dir: command.Dir, - Env: isolatedGitEnvironment(os.Environ(), command.Dir), StdoutLimitBytes: command.MaxOutputBytes, + Argv: append([]string{"git"}, args...), Dir: command.Dir, + Env: environment, StdoutLimitBytes: command.MaxOutputBytes, }, 5*time.Second) output := append(append([]byte(nil), result.Stdout...), result.Stderr...) if err != nil { @@ -59,9 +71,15 @@ func (OSRunner) Run(ctx context.Context, command Command) ([]byte, error) { processadapter.IsExactStdoutLimitExitCode(err, 141) { return nil, processadapter.ErrStdoutLimitExceeded } + if command.NativeCredentials { + return nil, fmt.Errorf("native Git transport failed: %w", ctxOrError(ctx, err)) + } return nil, fmt.Errorf("git %s failed: %w: %s", strings.Join(command.Args, " "), err, strings.TrimSpace(string(output))) } if result.ExitCode != 0 { + if command.NativeCredentials { + return nil, fmt.Errorf("native Git authentication or transport failed (exit %d); configure Git credentials or SSH access and retry", result.ExitCode) + } return nil, fmt.Errorf("git %s failed with exit code %d: %s", strings.Join(command.Args, " "), result.ExitCode, strings.TrimSpace(string(output))) } return output, nil @@ -122,6 +140,10 @@ func (acquirer Acquirer) DiscoverGitHubPackages(ctx context.Context, repository, if acquirer.URLForRepo != nil { url = acquirer.URLForRepo(repository) } + return acquirer.discoverGitPackages(ctx, url, repository, revision, false) +} + +func (acquirer Acquirer) discoverGitPackages(ctx context.Context, url, repository, revision string, native bool) ([]string, error) { tempRoot, err := os.MkdirTemp(acquirer.TempRoot, "agentplugins-discovery-*") if err != nil { return nil, gitAcquisitionFailure(repository, revision, "create temporary discovery workspace") @@ -129,7 +151,8 @@ func (acquirer Acquirer) DiscoverGitHubPackages(ctx context.Context, repository, defer func() { _ = os.RemoveAll(tempRoot) }() repoRoot := filepath.Join(tempRoot, "repository") run := func(args ...string) ([]byte, error) { - return acquirer.runner().Run(ctx, Command{Dir: tempRoot, Args: args}) + transport := native && len(args) > 2 && args[2] == "fetch" + return acquirer.runner().Run(ctx, Command{Dir: tempRoot, Args: args, NativeCredentials: transport}) } if _, err := run("init", "--quiet", repoRoot); err != nil { return nil, gitAcquisitionFailure(repository, revision, "initialize discovery repository") @@ -142,7 +165,11 @@ func (acquirer Acquirer) DiscoverGitHubPackages(ctx context.Context, repository, return nil, gitAcquisitionFailure(repository, revision, "configure discovery repository") } } - if _, err := run("-C", repoRoot, "fetch", "--quiet", "--depth=1", "--filter=blob:none", "--no-tags", "--no-recurse-submodules", "origin", revision); err != nil { + fetchArgs := []string{"-C", repoRoot, "fetch", "--quiet", "--depth=1", "--no-tags", "--no-recurse-submodules", "origin", revision} + if !native { + fetchArgs = append(fetchArgs[:5], append([]string{"--filter=blob:none"}, fetchArgs[5:]...)...) + } + if _, err := run(fetchArgs...); err != nil { return nil, gitAcquisitionFailure(repository, revision, "fetch immutable revision for package discovery") } resolved, err := run("-C", repoRoot, "rev-parse", "--verify", "FETCH_HEAD^{commit}") @@ -223,6 +250,10 @@ func (acquirer Acquirer) AcquireLocal(ctx context.Context, sourceRoot string) (d } func (acquirer Acquirer) acquireGit(ctx context.Context, url, repository, revision, subpath string) (domain.PackageSnapshot, error) { + return acquirer.acquireGitTransport(ctx, url, repository, revision, subpath, false) +} + +func (acquirer Acquirer) acquireGitTransport(ctx context.Context, url, repository, revision, subpath string, native bool) (domain.PackageSnapshot, error) { tempRoot, err := os.MkdirTemp(acquirer.TempRoot, "agentplugins-git-*") if err != nil { return domain.PackageSnapshot{}, gitAcquisitionFailure(repository, revision, "create temporary workspace") @@ -230,7 +261,8 @@ func (acquirer Acquirer) acquireGit(ctx context.Context, url, repository, revisi defer func() { _ = os.RemoveAll(tempRoot) }() repoRoot := filepath.Join(tempRoot, "repository") run := func(args ...string) ([]byte, error) { - return acquirer.runner().Run(ctx, Command{Dir: tempRoot, Args: args}) + transport := native && len(args) > 2 && args[2] == "fetch" + return acquirer.runner().Run(ctx, Command{Dir: tempRoot, Args: args, NativeCredentials: transport}) } if _, err := run("init", "--quiet", repoRoot); err != nil { return domain.PackageSnapshot{}, gitAcquisitionFailure(repository, revision, "initialize repository") @@ -243,7 +275,11 @@ func (acquirer Acquirer) acquireGit(ctx context.Context, url, repository, revisi return domain.PackageSnapshot{}, gitAcquisitionFailure(repository, revision, "configure isolated repository") } } - if _, err := run("-C", repoRoot, "fetch", "--quiet", "--depth=1", "--filter=blob:none", "--no-tags", "--no-recurse-submodules", "origin", revision); err != nil { + fetchArgs := []string{"-C", repoRoot, "fetch", "--quiet", "--depth=1", "--no-tags", "--no-recurse-submodules", "origin", revision} + if !native { + fetchArgs = append(fetchArgs[:5], append([]string{"--filter=blob:none"}, fetchArgs[5:]...)...) + } + if _, err := run(fetchArgs...); err != nil { return domain.PackageSnapshot{}, gitAcquisitionFailure(repository, revision, "fetch immutable revision") } resolved, err := run("-C", repoRoot, "rev-parse", "--verify", "FETCH_HEAD^{commit}") diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go new file mode 100644 index 000000000..5d3b1cdd2 --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go @@ -0,0 +1,261 @@ +package sourceacquisition + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +func (acquirer Acquirer) gitEndpoint(source sourcespec.Git) (string, string, error) { + endpoint := source.URL + if endpoint == "" && source.Repository != "" { + endpoint = "https://github.com/" + source.Repository + ".git" + } + normalized, repository, err := sourcespec.NormalizeGitURL(endpoint) + if err != nil { + return "", "", err + } + if source.Repository != "" && !strings.EqualFold(repository, source.Repository) { + return "", "", errors.New("Git source repository disagrees with its endpoint") + } + return normalized, repository, nil +} + +func (acquirer Acquirer) directEndpoint(endpoint, repository string) string { + if repository != "" && acquirer.URLForRepo != nil { + return acquirer.URLForRepo(repository) + } + return endpoint +} + +func (acquirer Acquirer) AcquireGit(ctx context.Context, source sourcespec.Git, commit, expectedTreeDigest string) (domain.PackageSnapshot, error) { + endpoint, repository, err := acquirer.gitEndpoint(source) + if err != nil { + return domain.PackageSnapshot{}, err + } + if !sourcespec.ValidCommit(commit) || source.Browser != nil { + return domain.PackageSnapshot{}, errors.New("direct Git acquisition requires resolved immutable coordinates") + } + packagePath, err := sourcespec.NormalizeSubpath(source.Path) + if err != nil { + return domain.PackageSnapshot{}, err + } + snapshot, err := acquirer.acquireGitTransport(ctx, acquirer.directEndpoint(endpoint, repository), endpoint, commit, packagePath, true) + if err != nil { + if ctx.Err() != nil { + return domain.PackageSnapshot{}, ctx.Err() + } + return domain.PackageSnapshot{}, fmt.Errorf("acquire direct Git source: immutable fetch or package snapshot failed; verify native Git credentials and repository access") + } + if expectedTreeDigest != "" && snapshot.TreeDigest != expectedTreeDigest { + _ = packagedigest.Remove(snapshot) + return domain.PackageSnapshot{}, errors.New("direct Git package tree does not match the recorded digest") + } + snapshot.Source = domain.SourceIdentity{SourceKind: "git", SourceURL: endpoint, RequestedSource: gitInput(endpoint, commit, packagePath), CanonicalSource: sourcespec.Key(sourcespec.KindGit, endpoint, packagePath), Repository: repository, PackageSubpath: packagePath, ResolvedRevision: commit, SourceBindingHint: "direct-git"} + return snapshot, nil +} + +func gitInput(endpoint, selector, packagePath string) string { + value := endpoint + "#" + selector + if packagePath != "" { + value += "//" + packagePath + } + return value +} + +func (acquirer Acquirer) DiscoverGitPackages(ctx context.Context, source sourcespec.Git, commit string) ([]string, error) { + endpoint, repository, err := acquirer.gitEndpoint(source) + if err != nil { + return nil, err + } + if !sourcespec.ValidCommit(commit) || source.Browser != nil { + return nil, errors.New("package discovery requires a full immutable Git commit") + } + paths, err := acquirer.discoverGitPackages(ctx, acquirer.directEndpoint(endpoint, repository), endpoint, commit, true) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, errors.New("inspect direct Git source package roots failed; choose an explicit package path or verify native Git access") + } + return paths, nil +} + +func (acquirer Acquirer) ResolveGitSource(ctx context.Context, source sourcespec.Git) (ResolvedGit, error) { + if source.Browser != nil { + return acquirer.resolveBrowserGit(ctx, source) + } + selector, exact := source.Ref, true + if source.Commit != "" { + selector, exact = source.Commit, false + } else if selector == "" { + selector, exact = "default", false + } + resolution, err := acquirer.ResolveGit(ctx, source, selector, exact) + if err != nil { + return ResolvedGit{}, err + } + source.Commit = resolution.Commit + return ResolvedGit{Git: source, Resolution: resolution}, nil +} + +type directGitWorkspace struct { + root, repository string + acquirer Acquirer +} + +func (acquirer Acquirer) newDirectWorkspace(ctx context.Context) (directGitWorkspace, error) { + root, err := os.MkdirTemp(acquirer.TempRoot, "agentplugins-direct-ref-*") + if err != nil { + return directGitWorkspace{}, errors.New("create private Git transport workspace failed") + } + workspace := directGitWorkspace{root: root, repository: filepath.Join(root, "repository"), acquirer: acquirer} + if _, err := workspace.run(ctx, false, "init", "--quiet", workspace.repository); err != nil { + _ = os.RemoveAll(root) + return directGitWorkspace{}, errors.New("initialize private Git transport workspace failed") + } + return workspace, nil +} +func (workspace directGitWorkspace) run(ctx context.Context, native bool, args ...string) ([]byte, error) { + return workspace.acquirer.runner().Run(ctx, Command{Dir: workspace.root, Args: args, NativeCredentials: native, MaxOutputBytes: maxGitHubDiscoveryMetadataBytes}) +} +func (workspace directGitWorkspace) remote(ctx context.Context, args ...string) ([]byte, error) { + return workspace.run(ctx, true, append([]string{"-C", workspace.repository}, args...)...) +} + +func (acquirer Acquirer) ResolveGit(ctx context.Context, source sourcespec.Git, selector string, exactRef bool) (RefResolution, error) { + endpoint, repository, err := acquirer.gitEndpoint(source) + if err != nil { + return RefResolution{}, err + } + if !exactRef && sourcespec.ValidCommit(selector) { + return RefResolution{RequestedRef: selector, Commit: selector}, nil + } + if err := sourcespec.ValidateRef(selector); err != nil { + return RefResolution{}, err + } + workspace, err := acquirer.newDirectWorkspace(ctx) + if err != nil { + return RefResolution{}, err + } + defer func() { _ = os.RemoveAll(workspace.root) }() + endpoint = acquirer.directEndpoint(endpoint, repository) + if !exactRef && (selector == "default" || selector == "HEAD") { + output, err := workspace.remote(ctx, "ls-remote", "--symref", endpoint, "HEAD") + if err != nil { + return RefResolution{}, directRefError(ctx) + } + branch, commit, err := parseDefaultBranch(output) + if err != nil { + return RefResolution{}, err + } + return RefResolution{RequestedRef: selector, ResolvedRef: branch, Commit: commit, DefaultBranch: branch}, nil + } + patterns := []string{selector} + resolvedRef := selector + if !strings.HasPrefix(selector, "refs/") { + patterns = []string{"refs/heads/" + selector, "refs/tags/" + selector} + resolvedRef = "" + } else if !strings.HasPrefix(selector, "refs/heads/") && !strings.HasPrefix(selector, "refs/tags/") { + return RefResolution{}, errors.New("only branches and tags are supported Git refs") + } + for _, ref := range patterns { + if _, err := workspace.run(ctx, false, "-C", workspace.repository, "check-ref-format", ref); err != nil { + return RefResolution{}, errors.New("invalid exact Git ref") + } + } + output, err := workspace.remote(ctx, append([]string{"ls-remote", "--refs", endpoint}, patterns...)...) + if err != nil { + return RefResolution{}, directRefError(ctx) + } + match, err := selectExactGitRef(output, patterns, selector, resolvedRef) + if err != nil { + return RefResolution{}, err + } + commit, err := workspace.commit(ctx, endpoint, match) + if err != nil { + return RefResolution{}, err + } + return RefResolution{RequestedRef: selector, ResolvedRef: match.name, Commit: commit}, nil +} + +func (workspace directGitWorkspace) commit(ctx context.Context, endpoint string, ref remoteRef) (string, error) { + if strings.HasPrefix(ref.name, "refs/heads/") { + return ref.object, nil + } + if _, err := workspace.remote(ctx, "fetch", "--quiet", "--depth=1", "--no-tags", "--no-recurse-submodules", endpoint, ref.object); err != nil { + return "", directRefError(ctx) + } + output, err := workspace.run(ctx, false, "-C", workspace.repository, "rev-parse", "--verify", "FETCH_HEAD^{commit}") + if err != nil || !sourcespec.ValidCommit(strings.TrimSpace(string(output))) { + return "", errors.New("Git tag did not resolve to a commit") + } + return strings.TrimSpace(string(output)), nil +} + +func (acquirer Acquirer) resolveBrowserGit(ctx context.Context, source sourcespec.Git) (ResolvedGit, error) { + endpoint, repository, err := acquirer.gitEndpoint(source) + if err != nil { + return ResolvedGit{}, err + } + suffix := source.Browser.Suffix + first, remainder, _ := strings.Cut(suffix, "/") + if sourcespec.ValidCommit(first) { + source.Browser = nil + source.Commit = first + source.Path, err = sourcespec.NormalizeSubpath(remainder) + return ResolvedGit{Git: source, Resolution: RefResolution{RequestedRef: first, Commit: first}}, err + } + workspace, err := acquirer.newDirectWorkspace(ctx) + if err != nil { + return ResolvedGit{}, err + } + defer func() { _ = os.RemoveAll(workspace.root) }() + transportEndpoint := acquirer.directEndpoint(endpoint, repository) + output, err := workspace.remote(ctx, "ls-remote", "--refs", transportEndpoint, "refs/heads/*", "refs/tags/*") + if err != nil { + return ResolvedGit{}, directRefError(ctx) + } + type candidate struct { + ref remoteRef + path string + } + var candidates []candidate + for _, ref := range parseRefs(output) { + name := strings.TrimPrefix(strings.TrimPrefix(ref.name, "refs/heads/"), "refs/tags/") + if suffix != name && !strings.HasPrefix(suffix, name+"/") { + continue + } + packagePath := strings.TrimPrefix(strings.TrimPrefix(suffix, name), "/") + if _, err := sourcespec.NormalizeSubpath(packagePath); err == nil { + candidates = append(candidates, candidate{ref: ref, path: packagePath}) + } + } + if len(candidates) != 1 { + return ResolvedGit{}, errors.New("browser tree selector is absent or ambiguous; use the clone URL#refs/heads/branch//package") + } + selected := candidates[0] + commit, err := workspace.commit(ctx, transportEndpoint, selected.ref) + if err != nil { + return ResolvedGit{}, err + } + source.Browser = nil + source.Ref = selected.ref.name + source.Commit = commit + source.Path = selected.path + return ResolvedGit{Git: source, Resolution: RefResolution{RequestedRef: selected.ref.name, ResolvedRef: selected.ref.name, Commit: commit}}, nil +} + +func directRefError(ctx context.Context) error { + if ctx.Err() != nil { + return ctx.Err() + } + return errors.New("resolve direct Git ref failed; verify native Git credentials and repository access") +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go new file mode 100644 index 000000000..d7eced7fd --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go @@ -0,0 +1,139 @@ +package sourceacquisition + +import ( + "bytes" + "fmt" + "net" + "os" + "os/exec" + "os/user" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +func TestNativeSSHGitWithIsolatedKeysAndHostVerification(t *testing.T) { + sshd, err := exec.LookPath("sshd") + if err != nil { + t.Skip("native SSH fixture requires openssh-server") + } + root, _, commit := nativeFixtureRepository(t) + current, err := user.Current() + if err != nil { + t.Fatal(err) + } + fixture := newNativeSSHFixture(t, sshd, root, current.Username) + t.Setenv("GIT_SSH_COMMAND", "ssh -F "+fixture.clientConfig) + source := sourcespec.Git{URL: fmt.Sprintf("ssh://%s@127.0.0.1:%d%s", current.Username, fixture.port, filepath.Join(root, "repo.git"))} + acquirer := Acquirer{TempRoot: t.TempDir()} + resolved, err := acquirer.ResolveGitSource(t.Context(), source) + if err != nil { + t.Fatal(err) + } + if resolved.Resolution.Commit != commit { + t.Fatal("SSH resolved another commit") + } + snapshot, err := acquirer.AcquireGit(t.Context(), source, commit, "") + if err != nil { + t.Fatal(err) + } + _ = packagedigest.Remove(snapshot) + scp, err := sourcespec.Parse(current.Username + "@uap-source-fixture:" + filepath.Join(root, "repo.git")) + if err != nil { + t.Fatal(err) + } + snapshot, err = acquirer.AcquireGit(t.Context(), *scp.Git, commit, "") + if err != nil { + t.Fatal(err) + } + _ = packagedigest.Remove(snapshot) + if err := os.WriteFile(fixture.knownHosts, nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := acquirer.AcquireGit(t.Context(), source, commit, ""); err == nil { + t.Fatal("untrusted SSH host accepted") + } +} + +type nativeSSHFixture struct { + port int + clientConfig, knownHosts string +} + +func newNativeSSHFixture(t *testing.T, sshd, repositoryRoot, username string) nativeSSHFixture { + t.Helper() + dir := t.TempDir() + sshFixtureKey(t, filepath.Join(dir, "host")) + sshFixtureKey(t, filepath.Join(dir, "identity")) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := listener.Addr().(*net.TCPAddr).Port + if err := listener.Close(); err != nil { + t.Fatal(err) + } + clientKey, err := os.ReadFile(filepath.Join(dir, "identity.pub")) + if err != nil { + t.Fatal(err) + } + git, err := exec.LookPath("git") + if err != nil { + t.Fatal(err) + } + authorized := fmt.Sprintf("command=\"%s upload-pack '%s'\",no-port-forwarding,no-agent-forwarding,no-X11-forwarding,no-pty %s", git, filepath.Join(repositoryRoot, "repo.git"), clientKey) + sshFixtureWrite(t, filepath.Join(dir, "authorized_keys"), authorized) + config := fmt.Sprintf("Port %d\nListenAddress 127.0.0.1\nHostKey %s\nPidFile %s\nAuthorizedKeysFile %s\nStrictModes no\nPasswordAuthentication no\nKbdInteractiveAuthentication no\nUsePAM no\nPermitRootLogin forced-commands-only\nAllowUsers %s\nLogLevel ERROR\n", port, filepath.Join(dir, "host"), filepath.Join(dir, "pid"), filepath.Join(dir, "authorized_keys"), username) + serverConfig := filepath.Join(dir, "sshd_config") + sshFixtureWrite(t, serverConfig, config) + var logs bytes.Buffer + command := exec.Command(sshd, "-D", "-e", "-f", serverConfig) + command.Stdout = &logs + command.Stderr = &logs + if err := command.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = command.Process.Kill(); _ = command.Wait() }) + waitSSHFixture(t, port, &logs) + hostKey, err := os.ReadFile(filepath.Join(dir, "host.pub")) + if err != nil { + t.Fatal(err) + } + fields := strings.Fields(string(hostKey)) + knownHosts := filepath.Join(dir, "known_hosts") + sshFixtureWrite(t, knownHosts, fmt.Sprintf("[127.0.0.1]:%d %s %s\n", port, fields[0], fields[1])) + clientConfig := filepath.Join(dir, "ssh_config") + sshFixtureWrite(t, clientConfig, fmt.Sprintf("Host *\n HostName 127.0.0.1\n Port %d\n User %s\n IdentityFile %s\n IdentitiesOnly yes\n UserKnownHostsFile %s\n StrictHostKeyChecking yes\n BatchMode yes\n", port, username, filepath.Join(dir, "identity"), knownHosts)) + return nativeSSHFixture{port: port, clientConfig: clientConfig, knownHosts: knownHosts} +} + +func sshFixtureKey(t *testing.T, path string) { + t.Helper() + command := exec.CommandContext(t.Context(), "ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", path) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("fixture key generation: %v %s", err, output) + } +} +func sshFixtureWrite(t *testing.T, path, value string) { + t.Helper() + if err := os.WriteFile(path, []byte(value), 0o600); err != nil { + t.Fatal(err) + } +} +func waitSSHFixture(t *testing.T, port int, logs *bytes.Buffer) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + connection, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 100*time.Millisecond) + if err == nil { + _ = connection.Close() + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("disposable sshd did not start: %s", logs.String()) +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport.go new file mode 100644 index 000000000..e2814c276 --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport.go @@ -0,0 +1,93 @@ +package sourceacquisition + +import ( + "context" + "path/filepath" + "strings" + "time" + + processadapter "github.com/777genius/plugin-kit-ai/install/integrationctl/adapters/process" + legacyports "github.com/777genius/plugin-kit-ai/install/integrationctl/ports" +) + +// Native authentication belongs to the user, not the fetched repository. Keep +// HOME/config/agent and explicit authentication inputs while stripping routing, +// object injection, traces and arbitrary runtime Git configuration. +func nativeGitEnvironment(environ []string) []string { + clean := make([]string, 0, len(environ)+4) + for _, item := range environ { + key, _, _ := strings.Cut(item, "=") + upper := strings.ToUpper(key) + if strings.HasPrefix(upper, "GIT_") { + switch upper { + case "GIT_ASKPASS", "GIT_SSH", "GIT_SSH_COMMAND", "GIT_SSH_VARIANT", "GIT_SSL_CAINFO", "GIT_SSL_CAPATH": + default: + continue + } + } + if upper == "GCM_INTERACTIVE" || upper == "SSH_ASKPASS_REQUIRE" { + continue + } + clean = append(clean, item) + } + return append(clean, "GIT_TERMINAL_PROMPT=0", "GIT_LFS_SKIP_SMUDGE=1", "GCM_INTERACTIVE=Never", "SSH_ASKPASS_REQUIRE=never") +} + +func nativeGitOptions(workspace string) []string { + return []string{ + "-c", "core.hooksPath=" + filepath.Join(workspace, "disabled-hooks"), + "-c", "core.fsmonitor=false", "-c", "maintenance.auto=false", "-c", "gc.auto=0", + "-c", "fetch.recurseSubmodules=false", "-c", "submodule.recurse=false", + "-c", "http.sslVerify=true", "-c", "protocol.allow=never", + "-c", "protocol.https.allow=always", "-c", "protocol.ssh.allow=always", + } +} + +// Git's core.sshCommand is trusted user configuration. Read it without exposing +// its text, then append only a fixed batch option. No source text enters a shell +// command. Native Git still supplies/quotes the host and repository arguments. +func withBatchSSH(ctx context.Context, environ []string, dir string) []string { + command, variant, wrapper := "", "", false + for _, item := range environ { + key, value, _ := strings.Cut(item, "=") + switch key { + case "GIT_SSH_COMMAND": + command = value + case "GIT_SSH_VARIANT": + variant = value + case "GIT_SSH": + wrapper = value != "" + } + } + if wrapper && command == "" { + return environ + } // an explicit trusted wrapper owns its arguments + if command == "" { + result, err := (processadapter.OS{}).RunWithTreeExitGrace(ctx, legacyports.Command{Argv: []string{"git", "config", "--get", "core.sshCommand"}, Dir: dir, Env: environ, StdoutLimitBytes: 8192}, time.Second) + if err == nil && result.ExitCode == 0 { + command = strings.TrimSpace(string(result.Stdout)) + } + } + if command == "" { + command = "ssh" + } + if variant == "plink" || variant == "tortoiseplink" || variant == "putty" { + command += " -batch" + } else { + command += " -oBatchMode=yes" + } + clean := make([]string, 0, len(environ)+1) + for _, item := range environ { + if !strings.HasPrefix(item, "GIT_SSH_COMMAND=") { + clean = append(clean, item) + } + } + return append(clean, "GIT_SSH_COMMAND="+command) +} + +func ctxOrError(ctx context.Context, err error) error { + if ctx.Err() != nil { + return ctx.Err() + } + return err +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport_test.go new file mode 100644 index 000000000..1c8022c33 --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport_test.go @@ -0,0 +1,188 @@ +package sourceacquisition + +import ( + "context" + "encoding/pem" + "fmt" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +func TestNativeGitEnvironmentSeparatesAuthenticationFromRouting(t *testing.T) { + env := nativeGitEnvironment([]string{"HOME=/user", "SSH_AUTH_SOCK=/agent", "GIT_ASKPASS=/helper", "GIT_SSH_COMMAND=ssh -F /config", "GIT_SSL_CAINFO=/ca", "GIT_DIR=/evil", "GIT_WORK_TREE=/evil", "GIT_CONFIG_COUNT=1", "GIT_CONFIG_KEY_0=http.extraHeader", "GIT_CONFIG_VALUE_0=secret", "GIT_TRACE=/leak", "GIT_SSL_NO_VERIFY=1"}) + joined := strings.Join(env, "\n") + for _, wanted := range []string{"HOME=/user", "SSH_AUTH_SOCK=/agent", "GIT_ASKPASS=/helper", "GIT_SSH_COMMAND=ssh -F /config", "GIT_SSL_CAINFO=/ca", "GIT_TERMINAL_PROMPT=0", "GCM_INTERACTIVE=Never"} { + if !strings.Contains(joined, wanted) { + t.Fatalf("missing native auth input %s", wanted) + } + } + for _, forbidden := range []string{"GIT_DIR=", "GIT_WORK_TREE=", "GIT_CONFIG_", "GIT_TRACE=", "GIT_SSL_NO_VERIFY="} { + if strings.Contains(joined, forbidden) { + t.Fatalf("unsafe native environment %s", forbidden) + } + } + anonymous := strings.Join(isolatedGitEnvironment(env, "/private"), "\n") + if strings.Contains(anonymous, "SSH_AUTH_SOCK=") || strings.Contains(anonymous, "GIT_ASKPASS=") || strings.Contains(anonymous, "HOME=/user") { + t.Fatal("signed anonymous environment acquired native credentials") + } +} + +func nativeFixtureGit(t *testing.T, dir string, args ...string) string { + t.Helper() + command := exec.CommandContext(t.Context(), "git", args...) + command.Dir = dir + command.Env = isolatedGitEnvironment(os.Environ(), dir) + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("fixture git %s: %v: %s", strings.Join(args, " "), err, output) + } + return strings.TrimSpace(string(output)) +} + +func nativeFixtureRepository(t *testing.T) (root, work, commit string) { + t.Helper() + root = t.TempDir() + work = filepath.Join(root, "work") + if err := os.Mkdir(work, 0o700); err != nil { + t.Fatal(err) + } + nativeFixtureGit(t, work, "init", "--quiet") + for name, body := range map[string]string{"plugin.json": `{"$schema":"https://agent-plugins.org/schemas/1.0.0/plugin.schema.json","name":"transport-demo","version":"1.0.0"}`, "skills/demo/SKILL.md": "---\nname: transport-demo\ndescription: Transport fixture\n---\nDo not execute package payloads.\n", "payload": "inert bytes\n", ".gitattributes": "payload filter=trap\n"} { + target := filepath.Join(work, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + nativeFixtureGit(t, work, "add", ".") + nativeFixtureGit(t, work, "-c", "user.name=Source Fixture", "-c", "user.email=fixture@example.invalid", "commit", "--quiet", "-m", "fixture") + commit = nativeFixtureGit(t, work, "rev-parse", "HEAD") + nativeFixtureGit(t, root, "clone", "--quiet", "--bare", work, filepath.Join(root, "repo.git")) + return root, work, commit +} + +func nativeHTTPSFixture(t *testing.T, root string, private bool) *httptest.Server { + t.Helper() + git, err := exec.LookPath("git") + if err != nil { + t.Fatal(err) + } + backend := &cgi.Handler{Path: git, Args: []string{"http-backend"}, Env: []string{"GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}} + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if private { + user, password, ok := r.BasicAuth() + if !ok || user != "fixture" || password != "fixture-token-never-log" { + w.Header().Set("WWW-Authenticate", `Basic realm="fixture"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + } + backend.ServeHTTP(w, r) + })) + t.Cleanup(server.Close) + ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + caPath := filepath.Join(t.TempDir(), "ca.pem") + if err := os.WriteFile(caPath, ca, 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("GIT_SSL_CAINFO", caPath) + return server +} + +func TestNativeHTTPSGitPublicAndPrivateCredentialHelper(t *testing.T) { + for _, private := range []bool{false, true} { + t.Run(fmt.Sprint(private), func(t *testing.T) { + root, _, commit := nativeFixtureRepository(t) + server := nativeHTTPSFixture(t, root, private) + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", home) + endpoint := server.URL + "/repo.git" + source := sourcespec.Git{URL: endpoint} + acquirer := Acquirer{TempRoot: t.TempDir()} + if private { + _, err := acquirer.ResolveGit(t.Context(), source, "default", false) + if err == nil || strings.Contains(err.Error(), "fixture-token-never-log") { + t.Fatalf("unprovisioned private source accepted or leaked credentials: %v", err) + } + } + configureNativeFixtureCredentials(t, home, server.URL, private) + resolved, err := acquirer.ResolveGitSource(t.Context(), source) + if err != nil { + t.Fatal(err) + } + if resolved.Resolution.Commit != commit { + t.Fatalf("resolved %s, expected %s", resolved.Resolution.Commit, commit) + } + snapshot, err := acquirer.AcquireGit(t.Context(), source, commit, "") + if err != nil { + t.Fatal(err) + } + defer func() { _ = packagedigest.Remove(snapshot) }() + if snapshot.Source.SourceURL != endpoint || snapshot.Source.ResolvedRevision != commit || snapshot.Source.SourceKind != "git" { + t.Fatalf("wrong immutable identity %+v", snapshot.Source) + } + if _, err := os.Stat(filepath.Join(home, "filter-ran")); !os.IsNotExist(err) { + t.Fatal("package attributes executed a user smudge filter") + } + if _, err := os.Stat(filepath.Join(home, "hook-ran")); !os.IsNotExist(err) { + t.Fatal("fetch executed a configured hook") + } + if snapshot.TreeDigest == "" { + t.Fatal("unsealed native snapshot") + } + }) + } +} + +func configureNativeFixtureCredentials(t *testing.T, home, origin string, private bool) { + t.Helper() + hooks := filepath.Join(home, "hooks") + if err := os.Mkdir(hooks, 0o700); err != nil { + t.Fatal(err) + } + hook := "#!/bin/sh\ntouch '" + filepath.Join(home, "hook-ran") + "'\n" + if err := os.WriteFile(filepath.Join(hooks, "reference-transaction"), []byte(hook), 0o700); err != nil { + t.Fatal(err) + } + config := "[core]\n hooksPath = " + hooks + "\n[filter \"trap\"]\n smudge = touch '" + filepath.Join(home, "filter-ran") + "'; cat\n required = true\n" + if private { + config += "[credential \"" + origin + "\"]\n helper = \"!f() { printf 'username=fixture\\npassword=fixture-token-never-log\\n'; }; f\"\n useHttpPath = true\n" + } + if err := os.WriteFile(filepath.Join(home, ".gitconfig"), []byte(config), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestNativeGitBrowserSlashRefAndAmbiguity(t *testing.T) { + root, work, commit := nativeFixtureRepository(t) + nativeFixtureGit(t, work, "branch", "feature/a") + nativeFixtureGit(t, work, "push", "--quiet", filepath.Join(root, "repo.git"), "feature/a") + server := nativeHTTPSFixture(t, root, false) + t.Setenv("HOME", t.TempDir()) + acquirer := Acquirer{TempRoot: t.TempDir()} + source := sourcespec.Git{URL: server.URL + "/repo.git", Browser: &sourcespec.BrowserTree{Provider: "gitlab", Suffix: "feature/a/pkg"}} + resolved, err := acquirer.ResolveGitSource(context.Background(), source) + if err != nil { + t.Fatal(err) + } + if resolved.Git.Path != "pkg" || resolved.Git.Ref != "refs/heads/feature/a" || resolved.Git.Commit != commit || resolved.Git.Browser != nil { + t.Fatalf("browser coordinates changed %+v", resolved) + } + nativeFixtureGit(t, work, "tag", "feature") + nativeFixtureGit(t, work, "push", "--quiet", filepath.Join(root, "repo.git"), "refs/tags/feature") + if _, err := acquirer.ResolveGitSource(t.Context(), source); err == nil || !strings.Contains(err.Error(), "ambiguous") { + t.Fatalf("ambiguous browser branch/path accepted: %v", err) + } +} From aa4cf349f2203d552a299664bf9f608cca1bad84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Fri, 23 Oct 2026 19:11:08 +0300 Subject: [PATCH 05/25] fix(sources): enforce native transport policy and reviewed source grammar --- .../archive-dot-prefix-amendment.md | 29 ++ .../archive-redirect-amendment.md | 54 +++ .../sources-code-review-r1.md | 156 +++++++++ .../adapters/sourceacquisition/acquirer.go | 16 +- .../adapters/sourceacquisition/archive.go | 308 ++++++++++++++++++ .../sourceacquisition/archive_compat_test.go | 207 ++++++++++++ .../sourceacquisition/archive_download.go | 261 +++++++++++++++ .../sourceacquisition/archive_hostile_test.go | 241 ++++++++++++++ .../sourceacquisition/archive_limits_test.go | 106 ++++++ .../sourceacquisition/archive_policy.go | 123 +++++++ .../adapters/sourceacquisition/archive_tar.go | 239 ++++++++++++++ .../sourceacquisition/archive_test.go | 279 ++++++++++++++++ .../adapters/sourceacquisition/archive_zip.go | 260 +++++++++++++++ .../adapters/sourceacquisition/git_direct.go | 23 +- .../sourceacquisition/git_policy_test.go | 84 +++++ .../sourceacquisition/git_ssh_test.go | 25 +- .../sourceacquisition/git_transport.go | 30 +- .../sourceacquisition/git_transport_test.go | 4 +- .../agentplugins/environment/sources.go | 6 +- .../agentplugins/environment/sources_test.go | 26 +- .../agentplugins/sourcespec/git.go | 100 +++--- .../agentplugins/sourcespec/parse.go | 24 +- .../agentplugins/sourcespec/provider.go | 155 +++++---- .../agentplugins/sourcespec/source_test.go | 4 + 24 files changed, 2621 insertions(+), 139 deletions(-) create mode 100644 docs/plans/source-transports-e2e/archive-dot-prefix-amendment.md create mode 100644 docs/plans/source-transports-e2e/archive-redirect-amendment.md create mode 100644 docs/plans/source-transports-e2e/sources-code-review-r1.md create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/archive.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/archive_compat_test.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/archive_download.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/archive_hostile_test.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/archive_limits_test.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/archive_policy.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/archive_tar.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/archive_test.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/archive_zip.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/git_policy_test.go diff --git a/docs/plans/source-transports-e2e/archive-dot-prefix-amendment.md b/docs/plans/source-transports-e2e/archive-dot-prefix-amendment.md new file mode 100644 index 000000000..763426856 --- /dev/null +++ b/docs/plans/source-transports-e2e/archive-dot-prefix-amendment.md @@ -0,0 +1,29 @@ +# Archive members: one conventional leading dot prefix + +Owner authorization and both independent plan reviews completed on 2026-10-23. +This narrowly amends member-name handling in `sources-plan.md`; it does not +relax URL grammar or requested logical package paths. + +Ordinary `tar -czf package.tar.gz -C package .` output uses a root `./` directory +and members such as `./plugin.json` and `./skills/demo/SKILL.md`. ZIP, TAR and +effective PAX names may therefore have exactly one leading `./` removed before +the existing portable-path checks and collision map. The root marker is ignored +only for a directory and still counts toward archive limits. + +No unsafe input is cleaned into acceptance. After stripping one prefix, +residual dot/traversal or empty segments, absolute/drive/UNC/backslash/control +paths, links, special entries and nonportable names still fail. All explicit +names and implicit ancestor keys use the same effective name. Consequently +`./x` plus `x` is a duplicate, and `./X/a` plus `x/b` is a case collision. + +R1 by `project_core` accepted exactly this normalization and required shared +collision handling. R2 by `sources_lead` independently returned GO; the full +parent-workspace ledger is `research/sources-dot-prefix-review-r2.md`. The +parent recorded both GO decisions before implementation. + +`archive_compat_test.go` verifies real TAR/gzip-TAR/ZIP dot-prefixed packages, +logical root selection and executable inventory; aliases collide across both +formats. Regressions reject `././x`, `./../x`, `.//x`, `./C:/x`, `./` regular +files, repeated separators and prefixed links. Byte pins, wrapper rules and +strict archive endings remain unchanged. Preserved legacy bundle code is +unmodified. diff --git a/docs/plans/source-transports-e2e/archive-redirect-amendment.md b/docs/plans/source-transports-e2e/archive-redirect-amendment.md new file mode 100644 index 000000000..c1284da06 --- /dev/null +++ b/docs/plans/source-transports-e2e/archive-redirect-amendment.md @@ -0,0 +1,54 @@ +# Archive downloads: transient signed HTTPS redirects + +Owner authorization and both independent plan reviews completed on 2026-10-23. +This narrowly amends the downloader redirect policy in `sources-plan.md`. +The initial source grammar and persistent source identity remain unchanged. + +## Concrete interoperability evidence + +The parent worker followed the official public asset + +with an HTTPS HEAD request. The original URL had no query. Redirects reached +`release-assets.githubusercontent.com` over HTTPS with a query and final HTTP +200. Query values were neither printed nor saved. Rejecting every redirect +query therefore rejects an ordinary public release download despite a stable, +credential-free initial source. + +## Approved policy + +The initial and persisted archive URL must be credential-free HTTPS without a +query or fragment. The downloader may follow at most five transient redirects. +Each target must use verified HTTPS, have a nonempty host, have no userinfo, +fragment, backslash or control characters, and have a complete URL of at most +16 KiB. Signed path/query encoding is preserved without decoding, rewriting or +canonicalizing the signature input. + +Every redirect clears all request headers, including Referer, Authorization, +Cookie and custom headers. The downloader rebuilds only its nonsecret +Accept-Encoding header. It uses no cookie jar, Git helper, netrc, browser or +session credentials, and does not inherit custom redirect callbacks. Host CA +transport configuration does not authorize disabling TLS verification. + +The final target, Location header, response body and raw HTTP errors are never +included in user-visible errors or persistent metadata. Request failures report +only a sanitized phase/status. SourceURL, RequestedSource, the canonical source +key and lock intent keep the original stable endpoint and selected logical +package path. Original archive-byte SHA256 and tree/manifest verification still +apply. A redirected or repacked byte change does not evade a locked byte pin. + +## Independent reviews and verification + +R1 by `project_core` required a separate transient validator, bounded verified +HTTPS, no credential/header propagation, no secret persistence or diagnostics, +and original source/checksum preservation. R2 by `sources_lead` independently +returned GO under those conditions; its full parent-workspace ledger is +`research/sources-redirect-review-r2.md`. The parent recorded both GO decisions +before implementation. + +`archive_compat_test.go` exercises real cross-origin TLS redirects to an opaque +signed query. It checks exact query encoding, no credential/Referer forwarding, +original-only serialized provenance, sanitized HTTP failure, excessive target +length, downgrade/userinfo/fragment refusal and strict initial query rejection. +Other archive tests retain byte-pin mismatch, hop-limit and cleanup coverage. +These are native HTTPS/archive transport fixtures, separate from native client +activation proof. diff --git a/docs/plans/source-transports-e2e/sources-code-review-r1.md b/docs/plans/source-transports-e2e/sources-code-review-r1.md new file mode 100644 index 000000000..e46e17ea0 --- /dev/null +++ b/docs/plans/source-transports-e2e/sources-code-review-r1.md @@ -0,0 +1,156 @@ +# Direct source implementation review R1 + +Reviewer: `client_plans`, independent of source implementation. Date: 2026-10-23. +Reviewed committed S1/S2/S4 changes from `1a9b25de` through `ec7cc19b` in +`work/uap-sources`, using `git show` to exclude the concurrently edited CLI +integration. Current working files were also checked only to reproduce the +unchanged native transport policy and identify already pending parser fixes. +The separate archive implementation is excluded from this self-review; its R1 +is `archive-code-review-r1.md` by `project_core`. + +**Verdict: CHANGES REQUIRED.** Two native transport restrictions are bypassable +through ordinary user Git configuration. Three smaller offline grammar gaps +also require correction. Source lead received the concrete findings and has +started the native policy fixes. No source implementation files were changed +by this reviewer. + +## R1-1 — P1: URL-scoped Git configuration bypasses TLS verification + +`adapters/sourceacquisition/git_transport.go:nativeGitOptions` supplies generic +`-c http.sslVerify=true`. Git's URL-specific configuration can override that +setting. A disposable HOME containing: + +```gitconfig +[http "https://127.0.0.1:"] + sslVerify = false +``` + +allowed `OSRunner.Run` with `NativeCredentials:true` and native `git ls-remote` +to send an HTTP request to an untrusted local TLS fixture. The fixture returned +404, so Git then failed; the request counter was **1**, proving that certificate +verification had already been bypassed. No fixture CA was provisioned. Errors +remained sanitized, which is good but does not enforce verified transport. + +**Required fix:** use an authoritative per-command TLS verification override +that takes precedence over URL-scoped settings while preserving legitimate CA +configuration. Source lead proposes `GIT_SSL_NO_VERIFY=false`; verify it with a +real native Git regression against the same URL-specific false setting. Keep +signed anonymous acquisition isolated and do not inherit unsafe TLS bypasses. + +## R1-2 — P1: a specific protocol allowance bypasses the transport restriction + +The same helper supplies `protocol.allow=never` plus allowances for HTTPS and +SSH. A user's more specific `protocol.file.allow=always` remains effective. A +disposable user configuration containing a `url.file://...insteadOf` rewrite +from `https://fixture.invalid/` to a disposable local repository plus that file +allowance made the following native request succeed: + +```text +OSRunner.Run(NativeCredentials:true, + git -C fetch --quiet --depth=1 + https://fixture.invalid/repo.git HEAD) +``` + +Observed result: **success=true, err=nil**. The public input passed HTTPS +normalization, but the effective transport was forbidden local file access. +Only inert fixture Git content was used; no package commands ran. + +**Required fix:** hard-limit actual direct transport protocols per command, +independently of user Git configuration (`GIT_ALLOW_PROTOCOL=https:ssh` is the +proposed route). Preserve the existing explicit `URLForRepo` fixture seam only +as a narrow internal injection, not as input-derived permission. Test the +`insteadOf` + specific protocol allowance with the actual Git binary. Also +verify that an HTTPS remote's redirect cannot downgrade to HTTP; libcurl +redirects need their own concrete proof rather than assuming the initial Git +protocol allowlist covers them. + +## R1-3 — P2: explicit local archive names containing @ are misclassified + +`sourcespec/parse.go:Parse` checks shorthand `@` before explicit local archive +classification. Reproduced: + +```text +Parse("./package@1.0.zip") + kind="git", err="GitHub source requires owner/repo" +``` + +This is an ordinary explicit relative archive filename. The in-progress CLI's +`localArchiveInput` also relies on Parse, so the shared grammar prevents the +promised local archive acquisition rather than leaving this as a harmless +parser-only limitation. + +**Required fix:** honor explicit local path/archive precedence before shorthand +selection, preserving genuine `owner/repo@ref` and native SCP endpoints. Add +relative/absolute local archive names with `@` and shorthand/SCP regressions. +No filesystem probing is necessary inside the pure parser. + +## R1-4 — P2: new URL Git intent and acquisition use different path grammars + +`environment/model.go:Requirement.Validate` still applies legacy +`validateSubpath` to new generic URL Git coordinates. Actual probes showed: + +```text +GitSource{URL:"https://example.test/repo.git", Path:"CON"}: + Requirement.Validate() = nil + sourcespec.NormalizeSubpath() = error +GitSource{URL:"https://example.test/repo.git", Path:"a\\x01b"}: + Requirement.Validate() = nil + sourcespec.NormalizeSubpath() = error +``` + +Acquisition correctly refuses these inputs later, but new manifests and locks +can be admitted with paths the acquisition contract cannot consume. This +violates the accepted single normalized grammar for the new source variants. + +**Required fix:** validate new URL Git package paths with the shared strict +normalizer at intent/lock admission. Preserve already accepted historical +Repository/GitHub bytes and behavior rather than broadly tightening old locks. +Add matching manifest/model/acquisition validation regressions. + +## R1-5 — P2: decoded SSH usernames can contain controls + +`sourcespec/git.go:NormalizeGitURL` validates raw input and decoded path, but +its decoded username check omits control characters. Reproduced: + +```text +Parse("ssh://user%0Aname@example.test/repo") + kind="git", normalized URL retained, err=nil +``` + +The accepted endpoint policy rejects controls; percent encoding must not bypass +that check in the login component. This is a validation gap, without an observed +command injection claim. + +**Required fix:** reject decoded username controls (and equivalent unsafe +userinfo text) before any native command or persistent identity. Keep valid SSH +user routing and the narrow Azure username-only HTTPS normalization unchanged. + +## Component ledger + +| Component | Assessment | +| --- | --- | +| S1 pure typed grammar and stable tuple keys | Structure GO; R1-3/R1-5 require fixes. Userless SCP dispatch absent in frozen ec7cc19b is already fixed in the observed pending parser; retain its regression in the next commit. | +| Provider repository/tree parsing | GO within reviewed provider grammar: transient BrowserTree, slash-ref resolution, explicit ambiguity refusal, known query consumption, archive-before-provider classification. | +| S2 separate native fetch and isolated checkout | Separation GO; real native fixtures exercise HTTPS credentials and SSH host verification without calling fixtures native-client proof. R1-1/R1-2 block transport qualification. | +| Exact revisions, tag peeling and browser ambiguity | GO: immutable commit validation, native ref enumeration, tag fetch/peeling, final coordinate persistence. | +| Signed anonymous GitHub boundary | Existing anonymous methods and isolated environment remain separate; do not broaden Directory/Discovery credentials. | +| S4 additive intent/lock variants | GO after R1-4: explicit three-way source counting, archive pins, moving update rule, requested selectors, graph edges and immutable revisions. | +| Old lock/key compatibility | GO by inspection: optional fields omit for old values, canonical GitHub remains Repository in environment intent, old source-key and manifest digest formulas remain intact, old strict readers reject new fields. | +| Dependency manifest/editing | GO after shared grammar fixes: source mixtures and inappropriate fields fail, no credential-bearing URL text is echoed in tested semantic failures, source location reporting preserved, archive values cloned. | +| Domain provenance fields | Additive fields GO; complete persistence/copy/cache reconstruction is S5 and requires the later integration review. | +| S5 CLI/store/lifecycle integration | Outside this frozen review; concurrently edited files were not judged as finished implementation. Actual end-to-end source journeys and both source-cache population orders remain required. | + +## Verification and boundaries + +The two P1 observations used the actual installed Git binary through the +production OSRunner, disposable HOME/config/repositories and a real untrusted +loopback TLS server. No network service outside loopback was contacted, no +external repository or package code was executed, and no user configuration was +modified. Probe programs were temporary review-only files outside the worktree. +Pure grammar/model probes used the shared current definitions; the reviewed +policies/functions were compared with ec7cc19b before attributing findings. + +Focused source model/parser tests and the new native regressions must pass on +the corrective commit before R1 GO. Parent owns independent R2 routing, +publication and the later cumulative S5 review. Do not interpret this review as +an executable release or native agent-client runtime qualification. diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go index 2439edee7..4bc3837ab 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go @@ -39,6 +39,8 @@ type Command struct { MaxOutputBytes int // NativeCredentials is restricted to direct user-selected Git transport commands. NativeCredentials bool + // mappedLocal is restricted to the explicit URLForRepo test fixture seam. + mappedLocal bool } type Runner interface { @@ -59,7 +61,15 @@ func (OSRunner) Run(ctx context.Context, command Command) ([]byte, error) { defer cancel() environment = nativeGitEnvironment(os.Environ()) environment = withBatchSSH(ctx, environment, command.Dir) - args = append(nativeGitOptions(command.Dir), args...) + options, policyErr := nativeVerificationOptions(ctx, environment, command) + if policyErr != nil { + return nil, policyErr + } + if command.mappedLocal { + options = append(options, "-c", "protocol.file.allow=always") + environment = append(environment, "GIT_ALLOW_PROTOCOL=https:ssh:file") + } + args = append(options, args...) } result, err := (processadapter.OS{}).RunWithTreeExitGrace(ctx, legacyports.Command{ Argv: append([]string{"git"}, args...), Dir: command.Dir, @@ -152,7 +162,7 @@ func (acquirer Acquirer) discoverGitPackages(ctx context.Context, url, repositor repoRoot := filepath.Join(tempRoot, "repository") run := func(args ...string) ([]byte, error) { transport := native && len(args) > 2 && args[2] == "fetch" - return acquirer.runner().Run(ctx, Command{Dir: tempRoot, Args: args, NativeCredentials: transport}) + return acquirer.runner().Run(ctx, Command{Dir: tempRoot, Args: args, NativeCredentials: transport, mappedLocal: transport && acquirer.mappedLocalEndpoint(url)}) } if _, err := run("init", "--quiet", repoRoot); err != nil { return nil, gitAcquisitionFailure(repository, revision, "initialize discovery repository") @@ -262,7 +272,7 @@ func (acquirer Acquirer) acquireGitTransport(ctx context.Context, url, repositor repoRoot := filepath.Join(tempRoot, "repository") run := func(args ...string) ([]byte, error) { transport := native && len(args) > 2 && args[2] == "fetch" - return acquirer.runner().Run(ctx, Command{Dir: tempRoot, Args: args, NativeCredentials: transport}) + return acquirer.runner().Run(ctx, Command{Dir: tempRoot, Args: args, NativeCredentials: transport, mappedLocal: transport && acquirer.mappedLocalEndpoint(url)}) } if _, err := run("init", "--quiet", repoRoot); err != nil { return domain.PackageSnapshot{}, gitAcquisitionFailure(repository, revision, "initialize repository") diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive.go new file mode 100644 index 000000000..496d5a1ca --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive.go @@ -0,0 +1,308 @@ +package sourceacquisition + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +// AcquireArchive acquires one complete standard package without running its +// content. The caller owns the returned sealed snapshot via packagedigest.Remove; +// the original stream and extraction tree never outlive this call. +func (acquirer Acquirer) AcquireArchive(ctx context.Context, source sourcespec.Archive, expectedArchiveSHA256, expectedTreeDigest string) (domain.PackageSnapshot, error) { + var empty domain.PackageSnapshot + coordinates, err := archiveCoordinatesFor(source, expectedArchiveSHA256, expectedTreeDigest) + if err != nil { + return empty, err + } + ctx, cancel := context.WithTimeout(ctx, archiveTimeout) + defer cancel() + workspace, err := os.MkdirTemp(acquirer.TempRoot, "agentplugins-archive-*") + if err != nil { + return empty, errors.New("create private archive workspace") + } + defer func() { _ = os.RemoveAll(workspace) }() + filename, format, checksum, err := acquirer.archiveRetain(ctx, coordinates, workspace) + if err != nil { + return empty, err + } + extractionRoot := filepath.Join(workspace, "extracted") + executables, err := archiveUnpack(ctx, filename, extractionRoot, format) + if err != nil { + return empty, err + } + selected, logicalPath, err := archiveSelectPackage(extractionRoot, coordinates.packagePath) + if err != nil { + return empty, err + } + return acquirer.archiveSeal(ctx, coordinates, checksum, extractionRoot, selected, logicalPath, executables) +} + +type archiveCoordinates struct { + endpoint, packagePath, bytePin, treePin string + explicit bool +} + +func archiveCoordinatesFor(source sourcespec.Archive, bytePin, treePin string) (archiveCoordinates, error) { + var empty archiveCoordinates + packagePath, err := sourcespec.NormalizeSubpath(source.Path) + if err != nil { + return empty, err + } + for _, checksum := range []string{source.SHA256, bytePin} { + if checksum != "" && !sourcespec.ValidSHA256(checksum) { + return empty, errors.New("archive checksum requires 64 lowercase hex characters") + } + } + if source.SHA256 != "" && bytePin != "" && source.SHA256 != bytePin { + return empty, errors.New("requested and locked archive checksums disagree") + } + if bytePin == "" { + bytePin = source.SHA256 + } + if treePin != "" && (!strings.HasPrefix(treePin, "sha256:") || !sourcespec.ValidSHA256(strings.TrimPrefix(treePin, "sha256:"))) { + return empty, errors.New("archive tree pin requires a sha256 digest") + } + endpoint, remote, err := archiveEndpoint(source.URL) + if err != nil { + return empty, err + } + return archiveCoordinates{endpoint: endpoint, packagePath: packagePath, bytePin: bytePin, treePin: treePin, + explicit: !remote || archiveExplicit(endpoint) || bytePin != ""}, nil +} + +func archiveEndpoint(endpoint string) (string, bool, error) { + remote := strings.Contains(endpoint, "://") + var err error + if remote { + endpoint, err = sourcespec.NormalizeArchiveURL(endpoint) + } else if endpoint != "" { + endpoint, err = filepath.Abs(endpoint) + endpoint = filepath.Clean(endpoint) + } else { + err = errors.New("archive source is empty") + } + if err != nil { + return "", false, errors.New("archive source requires a public HTTPS endpoint or explicit local file") + } + return endpoint, remote, nil +} + +func (acquirer Acquirer) archiveRetain(ctx context.Context, coordinates archiveCoordinates, workspace string) (string, archiveFormat, string, error) { + filename := filepath.Join(workspace, "download") + format, checksum, err := acquirer.archiveDownload(ctx, coordinates.endpoint, filename, coordinates.explicit) + if err != nil { + return "", format, "", err + } + if coordinates.bytePin != "" && checksum != coordinates.bytePin { + return "", format, "", errors.New("archive original-byte SHA256 does not match the requested or locked checksum") + } + if format == archiveGzip { + tarFilename := filepath.Join(workspace, "stream.tar") + if err := archiveGunzip(ctx, filename, tarFilename); err != nil { + return "", format, "", err + } + filename, format = tarFilename, archiveTAR + } + return filename, format, checksum, nil +} + +func archiveUnpack(ctx context.Context, filename, extractionRoot string, format archiveFormat) ([]string, error) { + file, err := os.Open(filename) + if err != nil { + return nil, errors.New("open retained archive stream") + } + defer func() { _ = file.Close() }() + info, err := file.Stat() + if err != nil { + return nil, errors.New("inspect retained archive stream") + } + if err := os.Mkdir(extractionRoot, 0o700); err != nil { + return nil, errors.New("create private archive extraction root") + } + root, err := os.OpenRoot(extractionRoot) + if err != nil { + return nil, errors.New("open contained archive extraction root") + } + defer func() { _ = root.Close() }() + output := &archiveExtraction{root: root, policy: newArchivePolicy()} + switch format { + case archiveZIP: + err = archiveExtractZIP(ctx, file, info.Size(), output) + case archiveTAR: + err = archiveExtractTAR(ctx, file, info.Size(), output) + default: + err = errors.New("unsupported archive format") + } + return output.executables, err +} + +func (acquirer Acquirer) archiveSeal(ctx context.Context, coordinates archiveCoordinates, checksum, extractionRoot, selected, logicalPath string, executablePaths []string) (domain.PackageSnapshot, error) { + source := domain.SourceIdentity{ + SourceKind: "archive", SourceURL: coordinates.endpoint, ArchiveSHA256: checksum, + RequestedSource: coordinates.endpoint, CanonicalSource: sourcespec.Key(sourcespec.KindArchive, coordinates.endpoint, logicalPath), + PackageSubpath: logicalPath, ResolvedRevision: "sha256:" + checksum, SourceBindingHint: "direct-archive", + } + prefix, err := filepath.Rel(extractionRoot, selected) + if err != nil { + return domain.PackageSnapshot{}, errors.New("resolve selected archive package") + } + prefix = filepath.ToSlash(prefix) + if prefix == "." { + prefix = "" + } else { + prefix += "/" + } + executables := make([]string, 0) + for _, executable := range executablePaths { + if strings.HasPrefix(executable, prefix) { + executables = append(executables, strings.TrimPrefix(executable, prefix)) + } + } + snapshot, err := acquirer.digester().SnapshotWithExecutables(ctx, selected, source, executables) + if err != nil { + return domain.PackageSnapshot{}, fmt.Errorf("seal acquired archive package: %w", err) + } + if coordinates.treePin != "" && snapshot.TreeDigest != coordinates.treePin { + _ = packagedigest.Remove(snapshot) + return domain.PackageSnapshot{}, errors.New("archive package tree digest does not match the locked digest") + } + snapshot.AcquiredAt = acquirer.now() + return snapshot, nil +} + +type archiveExtraction struct { + root *os.Root + policy *archivePolicy + executables []string +} + +func (output *archiveExtraction) write(ctx context.Context, raw string, directory bool, size int64, executable bool, body io.Reader) error { + name, err := output.policy.entry(raw, directory, size) + if err != nil { + return err + } + if directory { + if name != "" { + if err := output.root.MkdirAll(filepath.FromSlash(name), 0o700); err != nil { + return errors.New("create archive directory") + } + } + var probe [1]byte + if n, err := body.Read(probe[:]); n != 0 || err != io.EOF { + return errors.New("invalid archive directory payload or checksum") + } + return nil + } + if err := ctx.Err(); err != nil { + return err + } + if err := output.root.MkdirAll(filepath.FromSlash(path.Dir(name)), 0o700); err != nil { + return errors.New("create archive file parent") + } + // No links are ever created in this private root. Rooted exclusive writes + // protect containment and reject overwrites independently of lexical policy. + file, err := output.root.OpenFile(filepath.FromSlash(name), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return errors.New("create contained archive file") + } + n, copyErr := io.Copy(file, io.LimitReader(&archiveContextReader{ctx: ctx, reader: body}, size+1)) + closeErr := file.Close() + if copyErr != nil || closeErr != nil || n != size { + if ctx.Err() != nil { + return ctx.Err() + } + return errors.New("archive payload size/checksum disagrees with its header") + } + if executable { + output.executables = append(output.executables, name) + } + return nil +} + +func archiveSelectPackage(extracted, requested string) (string, string, error) { + logicalRoot := extracted + if !archiveManifestPresent(logicalRoot) { + entries, err := os.ReadDir(extracted) + if err != nil { + return "", "", errors.New("inspect archive package roots") + } + if len(entries) == 1 && entries[0].IsDir() { + logicalRoot = filepath.Join(extracted, entries[0].Name()) + } + } + if requested != "" { + selected := filepath.Join(logicalRoot, filepath.FromSlash(requested)) + if !archiveManifestAt(selected) { + return "", "", errors.New("selected archive path has no standard plugin.json") + } + return selected, requested, nil + } + if archiveManifestPresent(logicalRoot) { + return logicalRoot, "", nil + } + candidates, err := archiveDiscoverPackages(logicalRoot) + if err != nil { + return "", "", err + } + if len(candidates) == 1 { + return filepath.Join(logicalRoot, filepath.FromSlash(candidates[0])), candidates[0], nil + } + if len(candidates) == 0 { + return "", "", errors.New("archive contains no complete standard Agent Plugins package") + } + return "", "", fmt.Errorf("archive contains multiple packages; choose an explicit path: %s", strings.Join(candidates, ", ")) +} + +func archiveDiscoverPackages(logicalRoot string) ([]string, error) { + var candidates []string + err := filepath.WalkDir(logicalRoot, func(filename string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || entry.Name() != "plugin.json" { + return nil + } + parent := filepath.Dir(filename) + if !archivePackageShaped(parent) { + return nil + } + relative, err := filepath.Rel(logicalRoot, parent) + if err != nil { + return err + } + candidates = append(candidates, filepath.ToSlash(relative)) + if len(candidates) > 16 { + return errors.New("archive has more than 16 package candidates; choose an explicit path") + } + return nil + }) + sort.Strings(candidates) + return candidates, err +} + +func archiveManifestAt(root string) bool { + info, err := os.Stat(filepath.Join(root, "plugin.json")) + return err == nil && info.Mode().IsRegular() +} + +func archiveManifestPresent(root string) bool { + _, err := os.Lstat(filepath.Join(root, "plugin.json")) + return err == nil +} + +func archivePackageShaped(parent string) bool { + mcp, mcpErr := os.Stat(filepath.Join(parent, "mcp.json")) + skills, skillsErr := os.Stat(filepath.Join(parent, "skills")) + return mcpErr == nil && mcp.Mode().IsRegular() || skillsErr == nil && skills.IsDir() +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_compat_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_compat_test.go new file mode 100644 index 000000000..89b1e7e03 --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_compat_test.go @@ -0,0 +1,207 @@ +package sourceacquisition + +import ( + "archive/tar" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +func TestArchiveConventionalDotMembers(t *testing.T) { + entries := append([]archiveFixtureEntry{{name: "./", typeflag: tar.TypeDir}}, archivePackageEntries("./")...) + for name, payload := range map[string][]byte{ + "TAR": archiveFixtureTAR(t, entries), + "TGZ": archiveFixtureGzip(t, archiveFixtureTAR(t, entries)), + "ZIP": archiveFixtureZIP(t, archivePackageEntries("./"), ""), + } { + t.Run(name, func(t *testing.T) { + snapshot, err := (Acquirer{TempRoot: t.TempDir()}).AcquireArchive(context.Background(), sourcespec.Archive{URL: archiveLocalFile(t, payload)}, "", "") + if err != nil { + t.Fatal(err) + } + if snapshot.Source.PackageSubpath != "" || len(snapshot.ExecutableFiles) != 1 || snapshot.ExecutableFiles[0] != "bin/run.sh" { + t.Fatalf("effective root/member inventory = %+v", snapshot) + } + _ = packagedigest.Remove(snapshot) + }) + } + for _, unsafe := range []string{"././x", "./../x", ".//x", "./C:/x", "./a//b"} { + t.Run(unsafe, func(t *testing.T) { + archiveRejectLocal(t, archiveFixtureTAR(t, []archiveFixtureEntry{{name: unsafe, body: "x"}})) + }) + } + // TAR's writer refuses a slash-terminated regular filename, so build that + // hostile header directly and retain its valid checksum. + rootFile := archiveFixtureTAR(t, []archiveFixtureEntry{{name: "xx", body: "x"}}) + clear(rootFile[:100]) + copy(rootFile[:100], "./") + for index := 148; index < 156; index++ { + rootFile[index] = ' ' + } + checksum := 0 + for _, value := range rootFile[:512] { + checksum += int(value) + } + copy(rootFile[148:156], fmt.Sprintf("%06o\x00 ", checksum)) + archiveRejectLocal(t, rootFile) + for _, entries := range [][]archiveFixtureEntry{ + {{name: "./x", body: "a"}, {name: "x", body: "b"}}, + {{name: "./X/a", body: "a"}, {name: "x/b", body: "b"}}, + {{name: "./x/a", body: "a"}, {name: "x", body: "b"}}, + } { + archiveRejectLocal(t, archiveFixtureTAR(t, entries)) + archiveRejectLocal(t, archiveFixtureZIP(t, entries, "")) + } + archiveRejectLocal(t, archiveFixtureTAR(t, []archiveFixtureEntry{{name: "./link", typeflag: tar.TypeSymlink, link: "x"}})) +} + +func TestArchiveSignedRedirectIsTransientAndSanitized(t *testing.T) { + const signedQuery = "sig=fixture-token%2Fnever-persist%2B&part=a%2Fb" + payload := archiveFixtureZIP(t, archivePackageEntries(""), "") + var targetCalls atomic.Int32 + cdn := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + targetCalls.Add(1) + if request.URL.RawQuery != signedQuery { + t.Errorf("signed query encoding changed: %q", request.URL.RawQuery) + } + for _, name := range []string{"Referer", "Authorization", "Cookie", "X-Secret"} { + if request.Header.Get(name) != "" { + t.Errorf("redirect retained %s", name) + } + } + if request.URL.Path == "/denied" { + writer.WriteHeader(http.StatusForbidden) + _, _ = io.WriteString(writer, signedQuery) + return + } + _, _ = writer.Write(payload) + })) + defer cdn.Close() + location := cdn.URL + "/asset.tar?" + signedQuery // Redirect suffix is nonauthoritative. + stable := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + http.Redirect(writer, request, location, http.StatusFound) + })) + defer stable.Close() + temporary := t.TempDir() + acquirer := Acquirer{TempRoot: temporary, HTTPClient: stable.Client()} + source := sourcespec.Archive{URL: stable.URL + "/package.zip"} + snapshot, err := acquirer.AcquireArchive(context.Background(), source, "", "") + if err != nil { + t.Fatal(err) + } + metadata, _ := json.Marshal(snapshot) + if strings.Contains(string(metadata), "fixture-token") || strings.Contains(string(metadata), cdn.URL) || snapshot.Source.SourceURL != source.URL || snapshot.Source.RequestedSource != source.URL { + t.Fatalf("transient redirect leaked into metadata: %s", metadata) + } + _ = packagedigest.Remove(snapshot) + for name, target := range map[string]string{ + "denied": cdn.URL + "/denied?" + signedQuery, + "downgrade": "http://127.0.0.1:1/asset?" + signedQuery, + "userinfo": strings.Replace(cdn.URL, "https://", "https://secret@", 1) + "/asset", + "fragment": cdn.URL + "/asset#secret", + "length": cdn.URL + "/asset?sig=" + strings.Repeat("x", 16<<10), + } { + t.Run(name, func(t *testing.T) { + location = target + _, err := acquirer.AcquireArchive(context.Background(), source, "", "") + if err == nil || strings.Contains(err.Error(), "fixture-token") || strings.Contains(err.Error(), target) { + t.Fatalf("unsafe redirect or unsanitized error: %v", err) + } + archiveAssertEmpty(t, temporary) + }) + } + if targetCalls.Load() != 2 { + t.Fatalf("rejected redirect reached CDN: calls=%d", targetCalls.Load()) + } + if _, err := acquirer.AcquireArchive(context.Background(), sourcespec.Archive{URL: source.URL + "?" + signedQuery}, "", ""); err == nil { + t.Fatal("initial signed URL admitted to persistent intent") + } +} + +func TestArchiveCancellationAndDeclaredDownloadLimit(t *testing.T) { + payload := archiveFixtureZIP(t, archivePackageEntries(""), "") + started := make(chan struct{}) + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/large" { + writer.Header().Set("Content-Length", fmt.Sprint(archiveMaxBytes+1)) + _, _ = writer.Write(payload) + return + } + _, _ = writer.Write(payload[:512]) + writer.(http.Flusher).Flush() + close(started) + <-request.Context().Done() + })) + defer server.Close() + temporary := t.TempDir() + acquirer := Acquirer{TempRoot: temporary, HTTPClient: server.Client()} + _, err := acquirer.AcquireArchive(context.Background(), sourcespec.Archive{URL: server.URL + "/large"}, "", "") + if err == nil || errors.Is(err, ErrNotArchive) || !strings.Contains(err.Error(), "256 MiB") { + t.Fatalf("declared download limit not enforced: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { <-started; cancel() }() + _, err = acquirer.AcquireArchive(ctx, sourcespec.Archive{URL: server.URL + "/cancel"}, "", "") + if !errors.Is(err, context.Canceled) { + t.Fatalf("active download cancellation = %v", err) + } + archiveAssertEmpty(t, temporary) +} + +func TestArchiveNeverAcceptsTLSBypass(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + _, _ = writer.Write(archiveFixtureZIP(t, archivePackageEntries(""), "")) + })) + defer server.Close() + client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}} + _, err := (Acquirer{TempRoot: t.TempDir(), HTTPClient: client}).AcquireArchive(context.Background(), sourcespec.Archive{URL: server.URL + "/package.zip"}, "", "") + if err == nil { + t.Fatal("host-supplied TLS bypass was inherited") + } +} + +func TestArchiveInitialSuffixMustMatchRecognizedFormat(t *testing.T) { + entries := archivePackageEntries("") + tarBytes := archiveFixtureTAR(t, entries) + payloads := map[string][]byte{".zip": archiveFixtureZIP(t, entries, ""), ".tar": tarBytes, ".tar.gz": archiveFixtureGzip(t, tarBytes), ".tgz": archiveFixtureGzip(t, tarBytes)} + var payload []byte + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { _, _ = writer.Write(payload) })) + defer server.Close() + for suffix := range payloads { + for actual, bytes := range payloads { + if suffix == actual || (suffix == ".tgz" && actual == ".tar.gz") || (suffix == ".tar.gz" && actual == ".tgz") { + continue + } + t.Run(suffix+"-contains-"+actual, func(t *testing.T) { + payload = bytes + temporary := t.TempDir() + acquirer := Acquirer{TempRoot: temporary, HTTPClient: server.Client()} + local := filepath.Join(t.TempDir(), "package"+suffix) + if err := os.WriteFile(local, payload, 0o600); err != nil { + t.Fatal(err) + } + for _, endpoint := range []string{local, server.URL + "/package" + suffix} { + _, err := acquirer.AcquireArchive(context.Background(), sourcespec.Archive{URL: endpoint}, "", "") + if err == nil || errors.Is(err, ErrNotArchive) || !strings.Contains(err.Error(), "incompatible") { + t.Fatalf("initial suffix mismatch accepted or allowed fallback: %v", err) + } + archiveAssertEmpty(t, temporary) + } + }) + } + } +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_download.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_download.go new file mode 100644 index 000000000..102f5255f --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_download.go @@ -0,0 +1,261 @@ +package sourceacquisition + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + "unicode" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +const ( + archiveMaxBytes = int64(256 << 20) + archiveTimeout = 2 * time.Minute +) + +type archiveFormat uint8 + +const ( + archiveUnknown archiveFormat = iota + archiveZIP + archiveTAR + archiveGzip +) + +// archiveMagic commits the caller to archive semantics even if the rest of the +// stream is corrupt. Content-Type and a server's filename are not authoritative. +func archiveMagic(prefix []byte) archiveFormat { + if len(prefix) >= 4 && string(prefix[:2]) == "PK" { + switch string(prefix[2:4]) { + case "\x03\x04", "\x05\x06", "\x06\x06", "\x07\x08": + return archiveZIP + } + } + if len(prefix) >= 2 && prefix[0] == 0x1f && prefix[1] == 0x8b { + return archiveGzip + } + if len(prefix) >= 512 { + if string(prefix[257:262]) == "ustar" || archiveZero(prefix[:512]) { + return archiveTAR + } + // V7 TAR has no magic; its checksum is a positive recognition signal. + want, err := archiveTarNumber(prefix[148:156]) + var sum int64 + for i, value := range prefix[:512] { + if i >= 148 && i < 156 { + value = ' ' + } + sum += int64(value) + } + if err == nil && want == sum { + return archiveTAR + } + } + return archiveUnknown +} + +// archiveDownload writes at most one retained byte stream. Nonarchive HTTP +// responses can signal native Git fallback; every positively recognized archive +// failure remains an archive error, including oversized/truncated bodies. +func (acquirer Acquirer) archiveDownload(ctx context.Context, endpoint, destination string, explicit bool) (archiveFormat, string, error) { + body, length, err := acquirer.archiveOpen(ctx, endpoint, explicit) + if err != nil { + return archiveUnknown, "", err + } + defer func() { _ = body.Close() }() + reader := bufio.NewReader(&archiveContextReader{ctx: ctx, reader: body}) + prefix, peekErr := reader.Peek(512) + format := archiveMagic(prefix) + if format == archiveUnknown { + if ctx.Err() != nil { + return format, "", ctx.Err() + } + if peekErr != nil && !errors.Is(peekErr, io.EOF) && !errors.Is(peekErr, bufio.ErrBufferFull) { + return format, "", errors.New("read archive signature") + } + if !explicit { + return format, "", ErrNotArchive + } + return format, "", errors.New("source is not a supported ZIP, TAR or gzip TAR archive") + } + if length > archiveMaxBytes { + return format, "", errors.New("archive download exceeds 256 MiB") + } + if declared := archiveFilenameFormat(endpoint); declared != archiveUnknown && declared != format { + return format, "", errors.New("archive format is incompatible with the initial filename suffix") + } + checksum, err := archiveSave(ctx, reader, destination) + return format, checksum, err +} + +func (acquirer Acquirer) archiveOpen(ctx context.Context, endpoint string, explicit bool) (io.ReadCloser, int64, error) { + if strings.HasPrefix(endpoint, "https://") { + return acquirer.archiveHTTP(ctx, endpoint, explicit) + } + file, err := os.Open(endpoint) + if err != nil { + return nil, 0, errors.New("open local archive") + } + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() { + _ = file.Close() + return nil, 0, errors.New("local archive must be a regular file") + } + return file, info.Size(), nil +} + +func (acquirer Acquirer) archiveHTTP(ctx context.Context, endpoint string, explicit bool) (io.ReadCloser, int64, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody) + if err != nil { + return nil, 0, errors.New("create archive download request") + } + request.Header.Set("Accept-Encoding", "identity") + client := acquirer.archiveHTTPClient() + response, err := client.Do(request) + if err != nil { + client.CloseIdleConnections() + if ctx.Err() != nil { + return nil, 0, ctx.Err() + } + return nil, 0, errors.New("archive HTTPS request failed (check TLS, redirects and connectivity)") + } + if response.StatusCode != http.StatusOK { + _ = response.Body.Close() + client.CloseIdleConnections() + if !explicit { + return nil, 0, ErrNotArchive + } + return nil, 0, fmt.Errorf("archive download returned HTTP %d", response.StatusCode) + } + if response.Header.Get("Content-Encoding") != "" && response.Header.Get("Content-Encoding") != "identity" { + _ = response.Body.Close() + client.CloseIdleConnections() + return nil, 0, errors.New("archive download refuses HTTP content encoding") + } + return &archiveHTTPBody{ReadCloser: response.Body, client: client}, response.ContentLength, nil +} + +type archiveHTTPBody struct { + io.ReadCloser + client *http.Client +} + +func (body *archiveHTTPBody) Close() error { + err := body.ReadCloser.Close() + body.client.CloseIdleConnections() + return err +} + +func archiveSave(ctx context.Context, reader io.Reader, destination string) (string, error) { + file, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return "", errors.New("create private archive download") + } + hash := sha256.New() + n, copyErr := io.Copy(io.MultiWriter(file, hash), io.LimitReader(reader, archiveMaxBytes+1)) + closeErr := file.Close() + if n > archiveMaxBytes { + return "", errors.New("archive download exceeds 256 MiB") + } + if copyErr != nil || closeErr != nil { + if ctx.Err() != nil { + return "", ctx.Err() + } + return "", errors.New("archive download was incomplete") + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func (acquirer Acquirer) archiveHTTPClient() *http.Client { + // Retain a host-supplied CA/proxy transport, without inheriting cookies, + // authentication callbacks, redirect exceptions or unbounded timeouts. + client := &http.Client{Timeout: archiveTimeout} + if acquirer.HTTPClient != nil { + client.Transport = acquirer.HTTPClient.Transport + if timeout := acquirer.HTTPClient.Timeout; timeout > 0 && timeout < client.Timeout { + client.Timeout = timeout + } + } + if client.Transport == nil { + client.Transport = http.DefaultTransport + } + if transport, ok := client.Transport.(*http.Transport); ok { + transport = transport.Clone() + if transport.TLSClientConfig != nil { + transport.TLSClientConfig = transport.TLSClientConfig.Clone() + transport.TLSClientConfig.InsecureSkipVerify = false + } + client.Transport = transport + } + client.CheckRedirect = func(request *http.Request, via []*http.Request) error { + if len(via) > 5 { + return errors.New("archive redirect limit exceeded") + } + if err := archiveRedirectURL(request.URL); err != nil { + return err + } + clear(request.Header) + request.Header.Set("Accept-Encoding", "identity") + return nil + } + return client +} + +// Signed CDN coordinates exist only during this request. Initial intent stays +// query-free, and this validator deliberately preserves signed query encoding. +func archiveRedirectURL(target *url.URL) error { + value := target.String() + if len(value) > 16<<10 || target.Scheme != "https" || target.Hostname() == "" || target.User != nil || target.Fragment != "" || target.Opaque != "" || strings.ContainsRune(value, '\\') || strings.ContainsFunc(value, unicode.IsControl) || strings.ContainsFunc(target.Path, unicode.IsControl) { + return errors.New("archive redirects require bounded HTTPS URLs without userinfo, fragments or controls") + } + return nil +} + +type archiveContextReader struct { + ctx context.Context + reader io.Reader +} + +func (reader *archiveContextReader) Read(body []byte) (int, error) { + if err := reader.ctx.Err(); err != nil { + return 0, err + } + return reader.reader.Read(body) +} + +func archiveExplicit(endpoint string) bool { + u, err := url.Parse(endpoint) + return err == nil && sourcespec.HasArchiveSuffix(u.Path) +} + +func archiveFilenameFormat(endpoint string) archiveFormat { + filename := endpoint + if strings.HasPrefix(endpoint, "https://") { + parsed, err := url.Parse(endpoint) + if err != nil { + return archiveUnknown + } + filename = parsed.Path + } + filename = strings.ToLower(filename) + switch { + case strings.HasSuffix(filename, ".tar.gz"), strings.HasSuffix(filename, ".tgz"): + return archiveGzip + case strings.HasSuffix(filename, ".zip"): + return archiveZIP + case strings.HasSuffix(filename, ".tar"): + return archiveTAR + default: + return archiveUnknown + } +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_hostile_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_hostile_test.go new file mode 100644 index 000000000..23543e1d7 --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_hostile_test.go @@ -0,0 +1,241 @@ +package sourceacquisition + +import ( + "archive/tar" + "archive/zip" + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "net/http" + "net/http/cookiejar" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +func archiveRejectLocal(t *testing.T, payload []byte) { + t.Helper() + temporary := t.TempDir() + _, err := (Acquirer{TempRoot: temporary}).AcquireArchive(context.Background(), sourcespec.Archive{URL: archiveLocalFile(t, payload)}, "", "") + if err == nil || errors.Is(err, ErrNotArchive) { + t.Fatalf("hostile recognized archive accepted or allowed Git fallback: %v", err) + } + archiveAssertEmpty(t, temporary) +} + +func TestArchiveRejectsPathsAndCollisionsAcrossFormats(t *testing.T) { + cases := map[string][]archiveFixtureEntry{ + "traversal": {{name: "../escape", body: "x"}}, + "absolute": {{name: "/escape", body: "x"}}, + "drive": {{name: "C:/escape", body: "x"}}, + "UNC": {{name: `\\host\escape`, body: "x"}}, + "backslash": {{name: `a\b`, body: "x"}}, + "control": {{name: "a\x7fb", body: "x"}}, + "decomposed Unicode": {{name: "e\u0301", body: "x"}}, + "duplicate": {{name: "same", body: "x"}, {name: "same", body: "y"}}, + "case collision": {{name: "NAME", body: "x"}, {name: "name", body: "y"}}, + "Unicode fold collision": {{name: "Straße/a", body: "x"}, {name: "STRASSE/b", body: "y"}}, + "implicit ancestor collision": {{name: "Foo/a", body: "x"}, {name: "foo/b", body: "y"}}, + "file then directory": {{name: "a", body: "x"}, {name: "a/b", body: "y"}}, + "directory then file": {{name: "a/b", body: "x"}, {name: "a", body: "y"}}, + "deep": {{name: strings.Repeat("a/", 64) + "b", body: "x"}}, + "reserved Windows": {{name: "CON.txt", body: "x"}}, + "Git metadata": {{name: ".git/config", body: "x"}}, + } + for name, entries := range cases { + t.Run(name, func(t *testing.T) { + t.Run("ZIP", func(t *testing.T) { archiveRejectLocal(t, archiveFixtureZIP(t, entries, "")) }) + t.Run("TAR", func(t *testing.T) { archiveRejectLocal(t, archiveFixtureTAR(t, entries)) }) + }) + } +} + +func TestArchiveRejectsLinksAndSpecialEntries(t *testing.T) { + for _, kind := range []byte{tar.TypeSymlink, tar.TypeLink, tar.TypeFifo, tar.TypeChar, tar.TypeBlock, tar.TypeGNUSparse} { + t.Run(fmt.Sprintf("TAR-%d", kind), func(t *testing.T) { + archiveRejectLocal(t, archiveFixtureTAR(t, []archiveFixtureEntry{{name: "escape", typeflag: kind, link: "../outside"}})) + }) + } + for _, mode := range []os.FileMode{os.ModeSymlink, os.ModeNamedPipe, os.ModeDevice, os.ModeSocket} { + t.Run(fmt.Sprintf("ZIP-%d", mode), func(t *testing.T) { + archiveRejectLocal(t, archiveFixtureZIP(t, []archiveFixtureEntry{{name: "escape", body: "../outside", mode: mode | 0o777}}, "")) + }) + } +} + +func TestArchiveStrictTAREndingsAndPAX(t *testing.T) { + valid := archiveFixtureTAR(t, archivePackageEntries("")) + for name, payload := range map[string][]byte{ + "missing end": valid[:len(valid)-1024], + "one end block": valid[:len(valid)-512], + "truncated end": valid[:len(valid)-1], + "unaligned padding": append(bytes.Clone(valid), 0), + "nonzero tail": append(bytes.Clone(valid), bytes.Repeat([]byte{'x'}, 512)...), + "second TAR": append(bytes.Clone(valid), valid...), + } { + t.Run(name, func(t *testing.T) { archiveRejectLocal(t, payload) }) + } + entries := append([]archiveFixtureEntry{{name: "./", typeflag: tar.TypeDir}}, archivePackageEntries("")...) + entries[1].pax = map[string]string{"comment": "valid PAX metadata"} + padded := append(archiveFixtureTAR(t, entries), make([]byte, 1024)...) + snapshot, err := (Acquirer{TempRoot: t.TempDir()}).AcquireArchive(context.Background(), sourcespec.Archive{URL: archiveLocalFile(t, padded)}, "", "") + if err != nil { + t.Fatal(err) + } + _ = packagedigest.Remove(snapshot) + for name, pax := range map[string]map[string]string{ + "PAX traversal": {"path": "../escape"}, + "PAX sparse": {"GNU.sparse.major": "1"}, + } { + t.Run(name, func(t *testing.T) { + archiveRejectLocal(t, archiveFixtureTAR(t, []archiveFixtureEntry{{name: "safe", body: "x", pax: pax}})) + }) + } +} + +func TestArchiveStrictGzipBoundaries(t *testing.T) { + valid := archiveFixtureGzip(t, archiveFixtureTAR(t, archivePackageEntries(""))) + badCRC := bytes.Clone(valid) + badCRC[len(badCRC)-8] ^= 1 + for name, payload := range map[string][]byte{ + "second member": append(bytes.Clone(valid), valid...), + "outer tail": append(bytes.Clone(valid), 0), + "bad CRC": badCRC, + "truncated trailer": valid[:len(valid)-1], + "not TAR": archiveFixtureGzip(t, []byte("ordinary text")), + } { + t.Run(name, func(t *testing.T) { archiveRejectLocal(t, payload) }) + } +} + +func archiveMutateZIP(t *testing.T, source []byte, mutate func([]byte, []byte, []byte)) []byte { + t.Helper() + payload := bytes.Clone(source) + central := bytes.Index(payload, []byte("PK\x01\x02")) + end := bytes.LastIndex(payload, []byte("PK\x05\x06")) + if central < 0 || end < 0 { + t.Fatal("invalid fixture") + } + mutate(payload, payload[central:], payload[end:]) + return payload +} + +func TestArchiveStrictZIPBoundariesAndMetadata(t *testing.T) { + valid := archiveFixtureZIP(t, archivePackageEntries(""), "") + cases := map[string][]byte{ + "prefix": append([]byte("SFX"), valid...), + "tail": append(bytes.Clone(valid), 0), + "second ZIP": append(bytes.Clone(valid), valid...), + "truncated end": valid[:len(valid)-1], + } + mutators := map[string]func([]byte, []byte, []byte){ + "multiple disks": func(_, _, end []byte) { end[4] = 1 }, + "ZIP64": func(_, _, end []byte) { binary.LittleEndian.PutUint16(end[10:12], 65535) }, + "encryption": func(local, central, _ []byte) { local[6] |= 1; central[8] |= 1 }, + "unsupported method": func(local, central, _ []byte) { local[8] = 99; central[10] = 99 }, + "declared oversized": func(_, central, _ []byte) { binary.LittleEndian.PutUint32(central[24:28], 64<<20+1) }, + "actual size differs": func(_, central, _ []byte) { binary.LittleEndian.PutUint32(central[24:28], 1) }, + "corrupt payload": func(local, _, _ []byte) { local[40] ^= 1 }, + } + for name, mutate := range mutators { + cases[name] = archiveMutateZIP(t, valid, mutate) + } + for name, payload := range cases { + t.Run(name, func(t *testing.T) { archiveRejectLocal(t, payload) }) + } +} + +func TestArchivePreflightLimits(t *testing.T) { + largeTAR := archiveFixtureTAR(t, []archiveFixtureEntry{{name: "huge"}}) + copy(largeTAR[124:136], []byte("00400000001\x00")) // 512 MiB + 1: rejected before allocation/write. + archiveRejectLocal(t, largeTAR) + policy := newArchivePolicy() + for index := range 4 { + if _, err := policy.entry(fmt.Sprintf("file-%d", index), false, 64<<20); err != nil { + t.Fatal(err) + } + } + if _, err := policy.entry("too-much", false, 1); err == nil { + t.Fatal("expanded byte limit not enforced before writes") + } + policy = newArchivePolicy() + for index := range 10000 { + if _, err := policy.entry(fmt.Sprintf("file-%d", index), false, 0); err != nil { + t.Fatal(err) + } + } + if _, err := policy.entry("too-many", false, 0); err == nil { + t.Fatal("entry limit not enforced") + } +} + +func TestArchiveAutoClassificationAndRedirects(t *testing.T) { + valid := archiveFixtureZIP(t, archivePackageEntries(""), "") + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/html", "/explicit.zip": + _, _ = writer.Write([]byte("not a package")) + case "/private": + writer.WriteHeader(http.StatusUnauthorized) + case "/corrupt": + _, _ = writer.Write(valid[:len(valid)-1]) + case "/loop": + http.Redirect(writer, request, "/loop", http.StatusFound) + case "/downgrade": + http.Redirect(writer, request, "http://127.0.0.1:1/no", http.StatusFound) + case "/redirect": + http.SetCookie(writer, &http.Cookie{Name: "secret", Value: "do-not-forward"}) + http.Redirect(writer, request, "/valid", http.StatusFound) + case "/valid": + if request.Header.Get("Cookie") != "" || request.Header.Get("Authorization") != "" { + t.Error("redirect forwarded credentials") + } + _, _ = writer.Write(valid) + } + })) + defer server.Close() + client := server.Client() + client.Jar, _ = cookiejar.New(nil) + temporary := t.TempDir() + acquirer := Acquirer{TempRoot: temporary, HTTPClient: client} + for _, endpoint := range []string{"/html", "/private", "/corrupt", "/explicit.zip", "/loop", "/downgrade"} { + _, err := acquirer.AcquireArchive(context.Background(), sourcespec.Archive{URL: server.URL + endpoint}, "", "") + wantFallback := endpoint == "/html" || endpoint == "/private" + if err == nil || errors.Is(err, ErrNotArchive) != wantFallback { + t.Errorf("%s fallback=%v: %v", endpoint, errors.Is(err, ErrNotArchive), err) + } + archiveAssertEmpty(t, temporary) + } + snapshot, err := acquirer.AcquireArchive(context.Background(), sourcespec.Archive{URL: server.URL + "/redirect"}, "", "") + if err != nil { + t.Fatal(err) + } + _ = packagedigest.Remove(snapshot) + archiveAssertEmpty(t, temporary) +} + +func TestArchiveStoreZIPAndRootDirectoryMarker(t *testing.T) { + var buffer bytes.Buffer + writer := zip.NewWriter(&buffer) + for _, entry := range append([]archiveFixtureEntry{{name: "./", mode: os.ModeDir | 0o755}}, archivePackageEntries("")...) { + header := &zip.FileHeader{Name: entry.name, Method: zip.Store} + header.SetMode(entry.mode) + body, err := writer.CreateHeader(header) + if err != nil { + t.Fatal(err) + } + _, _ = body.Write([]byte(entry.body)) + } + _ = writer.Close() + snapshot, err := (Acquirer{TempRoot: t.TempDir()}).AcquireArchive(context.Background(), sourcespec.Archive{URL: archiveLocalFile(t, buffer.Bytes())}, "", "") + if err != nil { + t.Fatal(err) + } + _ = packagedigest.Remove(snapshot) +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_limits_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_limits_test.go new file mode 100644 index 000000000..0fed0c1eb --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_limits_test.go @@ -0,0 +1,106 @@ +package sourceacquisition + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/loader" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/specregistry" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +type archiveZeroReader struct{} + +func (archiveZeroReader) Read(body []byte) (int, error) { + clear(body) + return len(body), nil +} + +type archiveRoundTripper func(*http.Request) (*http.Response, error) + +func (roundTrip archiveRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + return roundTrip(request) +} + +func TestArchiveStreamingDownloadLimit(t *testing.T) { + // This component fixture deliberately omits Content-Length. The retained + // stream must be bounded by actual bytes, independently of server metadata. + client := &http.Client{Transport: archiveRoundTripper(func(*http.Request) (*http.Response, error) { + body := io.MultiReader(bytes.NewReader([]byte("PK\x03\x04")), io.LimitReader(archiveZeroReader{}, archiveMaxBytes)) + return &http.Response{StatusCode: http.StatusOK, ContentLength: -1, Header: make(http.Header), Body: io.NopCloser(body)}, nil + })} + temporary := t.TempDir() + _, err := (Acquirer{TempRoot: temporary, HTTPClient: client}).AcquireArchive(context.Background(), sourcespec.Archive{URL: "https://fixture.invalid/download"}, "", "") + if err == nil || errors.Is(err, ErrNotArchive) || !strings.Contains(err.Error(), "256 MiB") { + t.Fatalf("actual download bytes escaped the bound: %v", err) + } + archiveAssertEmpty(t, temporary) +} + +func TestArchiveStreamingGzipExpansionLimit(t *testing.T) { + var compressed bytes.Buffer + writer, err := gzip.NewWriterLevel(&compressed, gzip.BestSpeed) + if err != nil { + t.Fatal(err) + } + if _, err := io.CopyN(writer, archiveZeroReader{}, archiveMaxBytes+1); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + temporary := t.TempDir() + _, err = (Acquirer{TempRoot: temporary}).AcquireArchive(context.Background(), sourcespec.Archive{URL: archiveLocalFile(t, compressed.Bytes())}, "", "") + if err == nil || errors.Is(err, ErrNotArchive) || !strings.Contains(err.Error(), "expanded stream limit") { + t.Fatalf("actual gzip expansion escaped the bound: %v", err) + } + archiveAssertEmpty(t, temporary) +} + +func TestArchiveCandidateBoundAllowsExplicitSelection(t *testing.T) { + entries := make([]archiveFixtureEntry, 0, 17*3) + for index := range 17 { + entries = append(entries, archivePackageEntries(fmt.Sprintf("wrapper/plugins/p%02d/", index))...) + } + temporary := t.TempDir() + acquirer := Acquirer{TempRoot: temporary} + source := sourcespec.Archive{URL: archiveLocalFile(t, archiveFixtureZIP(t, entries, ""))} + if _, err := acquirer.AcquireArchive(context.Background(), source, "", ""); err == nil || !strings.Contains(err.Error(), "16 package candidates") { + t.Fatalf("unbounded package discovery: %v", err) + } + source.Path = "plugins/p16" + snapshot, err := acquirer.AcquireArchive(context.Background(), source, "", "") + if err != nil { + t.Fatal(err) + } + if snapshot.Source.PackageSubpath != source.Path { + t.Fatalf("explicit logical path = %q", snapshot.Source.PackageSubpath) + } + _ = packagedigest.Remove(snapshot) + archiveAssertEmpty(t, temporary) +} + +func TestArchiveInvalidRootManifestDirectoryNeverSelectsNested(t *testing.T) { + entries := append([]archiveFixtureEntry{{name: "plugin.json", typeflag: tar.TypeDir}}, archivePackageEntries("nested/")...) + snapshot, err := (Acquirer{TempRoot: t.TempDir()}).AcquireArchive(context.Background(), sourcespec.Archive{URL: archiveLocalFile(t, archiveFixtureTAR(t, entries))}, "", "") + if err != nil { + t.Fatal(err) + } + defer func() { _ = packagedigest.Remove(snapshot) }() + if snapshot.Source.PackageSubpath != "" { + t.Fatal("invalid root manifest object fell back to a nested package") + } + registry, _ := specregistry.New() + if _, err := (loader.Loader{Registry: registry}).LoadSnapshot(context.Background(), snapshot); err == nil { + t.Fatal("invalid root manifest directory bypassed standard validation") + } +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_policy.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_policy.go new file mode 100644 index 000000000..8cc6761d2 --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_policy.go @@ -0,0 +1,123 @@ +package sourceacquisition + +import ( + "errors" + "fmt" + "path" + "strings" + "unicode" + "unicode/utf8" + + "golang.org/x/text/cases" + "golang.org/x/text/unicode/norm" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/adapters/pathpolicy" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" +) + +type archivePath struct { + name string + directory bool + explicit bool +} + +type archivePolicy struct { + entries int + bytes int64 + paths map[string]archivePath +} + +func newArchivePolicy() *archivePolicy { + return &archivePolicy{paths: make(map[string]archivePath)} +} + +func (policy *archivePolicy) entry(raw string, directory bool, size int64) (string, error) { + policy.entries++ + if policy.entries > domain.DefaultMaxFiles { + return "", errors.New("archive exceeds 10000 entries") + } + if size < 0 || size > domain.DefaultMaxFileBytes || size > archiveMaxBytes-policy.bytes { + return "", errors.New("archive exceeds file (64 MiB) or expanded byte (256 MiB) limits") + } + if directory && size != 0 { + return "", errors.New("archive directory has nonzero size") + } + policy.bytes += size + // Conventional root markers still count toward entry and stream limits. + if directory && (raw == "./" || raw == ".") { + return "", nil + } + name, err := archiveNormalizePath(raw, directory) + if err != nil { + return "", err + } + if err := policy.registerPath(name, directory); err != nil { + return "", err + } + return name, nil +} + +func archiveNormalizePath(raw string, directory bool) (string, error) { + // Strip exactly one conventional prefix; residual dot/empty segments fail + // the unchanged portable-path checks rather than being path.Clean'ed away. + name := strings.TrimPrefix(raw, "./") + if directory { + name = strings.TrimSuffix(name, "/") + } + if name == "" || !utf8.ValidString(name) || !norm.NFC.IsNormalString(name) || path.IsAbs(name) || path.Clean(name) != name || strings.ContainsFunc(name, unicode.IsControl) { + return "", fmt.Errorf("archive contains non-portable path %q", raw) + } + parts := strings.Split(name, "/") + if len(parts) > domain.DefaultMaxDepth { + return "", errors.New("archive path exceeds depth 64") + } + for _, part := range parts { + if err := pathpolicy.ValidatePortablePathSegment(part); err != nil || strings.EqualFold(part, ".git") { + return "", fmt.Errorf("archive contains unsafe path %q", raw) + } + } + return name, nil +} + +func (policy *archivePolicy) registerPath(name string, directory bool) error { + parts := strings.Split(name, "/") + for index := range parts { + current := strings.Join(parts[:index+1], "/") + if err := policy.registerNode(current, index == len(parts)-1, directory); err != nil { + return err + } + if len(policy.paths) > domain.DefaultMaxFiles { + return errors.New("archive exceeds 10000 paths including implicit directories") + } + } + return nil +} + +func (policy *archivePolicy) registerNode(current string, leaf, directory bool) error { + key := cases.Fold().String(current) + prior, found := policy.paths[key] + if found && prior.name != current { + return fmt.Errorf("archive paths %q and %q collide by case or Unicode", prior.name, current) + } + if found && !prior.directory && (!leaf || directory) { + return fmt.Errorf("archive has file/directory conflict at %q", current) + } + if leaf && found && (prior.explicit || prior.directory && !directory) { + return fmt.Errorf("archive has duplicate or file/directory conflict at %q", current) + } + if leaf { + policy.paths[key] = archivePath{name: current, directory: directory, explicit: true} + } else if !found { + policy.paths[key] = archivePath{name: current, directory: true} + } + return nil +} + +func archiveZero(body []byte) bool { + for _, value := range body { + if value != 0 { + return false + } + } + return true +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_tar.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_tar.go new file mode 100644 index 000000000..d1e62dbec --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_tar.go @@ -0,0 +1,239 @@ +package sourceacquisition + +import ( + "archive/tar" + "bufio" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "math" + "os" + "strconv" + "strings" +) + +// archiveGunzip validates one member (including CRC/ISIZE) and preserves the +// exact next byte so concatenated members and outer trailers cannot disappear +// into gzip's buffering. Expansion is bounded before writing the TAR stream. +func archiveGunzip(ctx context.Context, compressed, destination string) error { + input, err := os.Open(compressed) + if err != nil { + return err + } + defer func() { _ = input.Close() }() + buffer := bufio.NewReader(&archiveContextReader{ctx: ctx, reader: input}) + reader, err := gzip.NewReader(buffer) + if err != nil { + return errors.New("invalid gzip archive header") + } + defer func() { _ = reader.Close() }() + reader.Multistream(false) + output, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + n, copyErr := io.Copy(output, io.LimitReader(reader, archiveMaxBytes+1)) + closeErr := output.Close() + if n > archiveMaxBytes { + return errors.New("gzip archive exceeds 256 MiB expanded stream limit") + } + if copyErr != nil || closeErr != nil { + return errors.New("invalid or truncated gzip archive (checksum/trailer required)") + } + if _, err := buffer.ReadByte(); err != io.EOF { + return errors.New("gzip archive has a second member or trailing bytes") + } + return nil +} + +// archiveValidateTAR audits raw framing before tar.Reader can hide metadata +// records or accept missing end blocks. PAX path/size overrides are interpreted +// again by tar.Reader during extraction; their effective paths use one policy. +func archiveValidateTAR(ctx context.Context, file *os.File, size int64) error { + if size > archiveMaxBytes || size%512 != 0 { + return errors.New("TAR stream exceeds its byte limit or has an unaligned ending") + } + state := archiveTARState{paxSize: -1} + for { + if err := ctx.Err(); err != nil { + return err + } + var block [512]byte + if _, err := file.ReadAt(block[:], state.offset); err != nil { + return errors.New("TAR requires two zero end blocks") + } + if archiveZero(block[:]) { + if state.offset+1024 > size || state.pendingMetadata { + return errors.New("TAR requires two zero end blocks after a complete entry") + } + return archiveZeroTail(ctx, io.NewSectionReader(file, state.offset, size-state.offset)) + } + if err := state.entry(file, block[:], size); err != nil { + return err + } + } +} + +type archiveTARState struct { + offset, expanded, paxSize int64 + entries int + pendingMetadata bool +} + +func (state *archiveTARState) entry(file *os.File, block []byte, size int64) error { + state.entries++ + if state.entries > 10000 { + return errors.New("TAR exceeds 10000 entries including metadata") + } + declared, err := archiveTarNumber(block[124:136]) + if err != nil || declared > 64<<20 { + return errors.New("TAR has invalid or oversized declared entry size") + } + typeflag := block[156] + metadata := typeflag == tar.TypeXHeader || typeflag == tar.TypeGNULongName + entrySize := declared + if !metadata { + if state.paxSize >= 0 { + entrySize = state.paxSize + } + state.paxSize = -1 + if err := archiveTARType(typeflag, entrySize); err != nil { + return err + } + } + if entrySize < 0 || entrySize > 64<<20 || entrySize > archiveMaxBytes-state.expanded { + return errors.New("TAR exceeds file or expanded byte limits") + } + state.expanded += entrySize + padded := (entrySize + 511) / 512 * 512 + if state.offset+512+padded > size { + return errors.New("truncated TAR entry") + } + if metadata { + if err := state.metadata(file, typeflag, entrySize); err != nil { + return err + } + } + state.pendingMetadata = metadata + state.offset += 512 + padded + return nil +} + +func archiveTARType(typeflag byte, size int64) error { + switch typeflag { + case tar.TypeReg, 0: + return nil + case tar.TypeDir: + if size == 0 { + return nil + } + return errors.New("TAR directory has nonzero size") + default: + return errors.New("TAR refuses links, sparse files, global metadata and special entries") + } +} + +func (state *archiveTARState) metadata(file *os.File, typeflag byte, size int64) error { + // Go's TAR parser has the same metadata bound. Enforce it before allocation. + if size > 1<<20 { + return errors.New("TAR metadata exceeds 1 MiB") + } + if typeflag != tar.TypeXHeader { + return nil + } + body := make([]byte, size) + if _, err := file.ReadAt(body, state.offset+512); err != nil { + return errors.New("truncated TAR metadata") + } + var err error + state.paxSize, err = archivePAXSize(string(body)) + return err +} + +func archivePAXSize(body string) (int64, error) { + size := int64(-1) + for body != "" { + prefix, _, ok := strings.Cut(body, " ") + length, err := strconv.Atoi(prefix) + if !ok || err != nil || length <= len(prefix)+3 || length > len(body) || body[length-1] != '\n' { + return 0, errors.New("invalid TAR PAX record") + } + key, value, ok := strings.Cut(body[len(prefix)+1:length-1], "=") + if !ok || key == "" || strings.Contains(key, "\x00") || strings.HasPrefix(key, "GNU.sparse") || key == "SCHILY.filetype" && value == "sparse" { + return 0, errors.New("unsupported TAR PAX metadata") + } + if key == "size" { + size, err = strconv.ParseInt(value, 10, 64) + if err != nil || size < 0 { + return 0, errors.New("invalid TAR PAX size") + } + } + body = body[length:] + } + return size, nil +} + +func archiveTarNumber(body []byte) (int64, error) { + if len(body) > 0 && body[0]&0x80 != 0 { + if body[0]&0x40 != 0 { + return 0, errors.New("negative TAR size") + } + value := int64(body[0] & 0x3f) + for _, digit := range body[1:] { + if value > (math.MaxInt64-int64(digit))/256 { + return 0, errors.New("overflowing TAR size") + } + value = value*256 + int64(digit) + } + return value, nil + } + text := strings.Trim(string(body), " \x00") + if text == "" { + return 0, nil + } + value, err := strconv.ParseInt(text, 8, 64) + if err != nil || value < 0 { + return 0, errors.New("invalid TAR size") + } + return value, nil +} + +func archiveZeroTail(ctx context.Context, reader io.Reader) error { + var buffer [32 << 10]byte + for { + if err := ctx.Err(); err != nil { + return err + } + n, err := reader.Read(buffer[:]) + if !archiveZero(buffer[:n]) { + return errors.New("TAR has nonzero trailing bytes or a second archive") + } + if err == io.EOF { + return nil + } + if err != nil { + return fmt.Errorf("read TAR end: %w", err) + } + } +} + +func archiveExtractTAR(ctx context.Context, file *os.File, size int64, output *archiveExtraction) error { + if err := archiveValidateTAR(ctx, file, size); err != nil { + return err + } + reader := tar.NewReader(&archiveContextReader{ctx: ctx, reader: io.NewSectionReader(file, 0, size)}) + for { + header, err := reader.Next() + if err == io.EOF { + return nil + } + if err != nil { + return errors.New("invalid TAR header or payload") + } + if err := output.write(ctx, header.Name, header.Typeflag == tar.TypeDir, header.Size, header.Mode&0o111 != 0, reader); err != nil { + return err + } + } +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_test.go new file mode 100644 index 000000000..1c9fba45f --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_test.go @@ -0,0 +1,279 @@ +package sourceacquisition + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "sync/atomic" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/loader" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/specregistry" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +type archiveFixtureEntry struct { + name, body, link string + mode os.FileMode + typeflag byte + pax map[string]string +} + +func archivePackageEntries(prefix string) []archiveFixtureEntry { + return []archiveFixtureEntry{ + {name: prefix + "plugin.json", body: `{"$schema":"https://agent-plugins.org/schemas/1.0.0/plugin.schema.json","name":"archive-demo","version":"1.0.0"}`}, + {name: prefix + "skills/demo/SKILL.md", body: "---\nname: demo\ndescription: Archive demo\n---\nKeep package content inert.\n"}, + {name: prefix + "bin/run.sh", body: "#!/bin/sh\ntouch SHOULD_NEVER_EXIST\n", mode: 0o4755}, + } +} + +func archiveFixtureZIP(t *testing.T, entries []archiveFixtureEntry, comment string) []byte { + t.Helper() + var buffer bytes.Buffer + writer := zip.NewWriter(&buffer) + if err := writer.SetComment(comment); err != nil { + t.Fatal(err) + } + for _, entry := range entries { + header := &zip.FileHeader{Name: entry.name, Method: zip.Deflate} + mode := entry.mode + if mode == 0 { + mode = 0o644 + } + header.SetMode(mode) + body, err := writer.CreateHeader(header) + if err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(body, entry.body); err != nil { + t.Fatal(err) + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return buffer.Bytes() +} + +func archiveFixtureTAR(t *testing.T, entries []archiveFixtureEntry) []byte { + t.Helper() + var buffer bytes.Buffer + writer := tar.NewWriter(&buffer) + for _, entry := range entries { + typeflag := entry.typeflag + if typeflag == 0 { + typeflag = tar.TypeReg + } + mode := int64(entry.mode.Perm()) + if mode == 0 { + mode = 0o644 + } + header := &tar.Header{Name: entry.name, Mode: mode, Typeflag: typeflag, Size: int64(len(entry.body)), Linkname: entry.link, PAXRecords: entry.pax} + if err := writer.WriteHeader(header); err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(writer, entry.body); err != nil { + t.Fatal(err) + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return buffer.Bytes() +} + +func archiveFixtureGzip(t *testing.T, body []byte) []byte { + t.Helper() + var buffer bytes.Buffer + writer := gzip.NewWriter(&buffer) + if _, err := writer.Write(body); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return buffer.Bytes() +} + +func archiveBytesSHA(body []byte) string { + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]) +} + +func archiveLocalFile(t *testing.T, body []byte) string { + t.Helper() + filename := filepath.Join(t.TempDir(), "package.bin") + if err := os.WriteFile(filename, body, 0o600); err != nil { + t.Fatal(err) + } + return filename +} + +func archiveAssertEmpty(t *testing.T, directory string) { + t.Helper() + entries, err := os.ReadDir(directory) + if err != nil || len(entries) != 0 { + t.Fatalf("temporary archive/snapshot leaked: %v, %v", entries, err) + } +} + +// This is actual HTTPS transport plus real archive parsing, standard loading +// and sealing. It is separate from native agent/client activation evidence. +func TestArchiveHTTPSFormatsReachStandardLoader(t *testing.T) { + entries := archivePackageEntries("release-old/") + tarBytes := archiveFixtureTAR(t, entries) + fixtures := map[string][]byte{ + "/package.zip": archiveFixtureZIP(t, entries, "declared ZIP comment"), + "/package.tar": tarBytes, + "/package.tar.gz": archiveFixtureGzip(t, tarBytes), + "/package.tgz": archiveFixtureGzip(t, tarBytes), + "/download": archiveFixtureZIP(t, entries, ""), + } + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.Header.Get("Authorization") != "" || request.Header.Get("Cookie") != "" || request.Header.Get("Accept-Encoding") != "identity" { + t.Error("archive request forwarded credentials or enabled content transformation") + } + writer.Header().Set("Content-Type", "text/plain") // Magic, not MIME, controls classification. + _, _ = writer.Write(fixtures[request.URL.Path]) + })) + defer server.Close() + registry, err := specregistry.New() + if err != nil { + t.Fatal(err) + } + var treeDigest string + for endpoint, payload := range fixtures { + t.Run(endpoint, func(t *testing.T) { + temporary := t.TempDir() + acquirer := Acquirer{TempRoot: temporary, HTTPClient: server.Client()} + snapshot, err := acquirer.AcquireArchive(context.Background(), sourcespec.Archive{URL: server.URL + endpoint}, "", "") + if err != nil { + t.Fatal(err) + } + envelope, err := (loader.Loader{Registry: registry}).LoadSnapshot(context.Background(), snapshot) + if err != nil || envelope.Manifest.Name != "archive-demo" || len(envelope.Skills) != 1 { + t.Fatalf("standard load = %+v, %v", envelope, err) + } + if snapshot.Source.ArchiveSHA256 != archiveBytesSHA(payload) || snapshot.Source.ResolvedRevision != "sha256:"+archiveBytesSHA(payload) || snapshot.Source.SourceURL != server.URL+endpoint || snapshot.Source.PackageSubpath != "" { + t.Fatalf("immutable provenance = %+v", snapshot.Source) + } + if treeDigest != "" && snapshot.TreeDigest != treeDigest { + t.Fatal("archive format changed the same tree's identity") + } + treeDigest = snapshot.TreeDigest + if !reflect.DeepEqual(snapshot.ExecutableFiles, []string{"bin/run.sh"}) { + t.Fatalf("executable intent = %v", snapshot.ExecutableFiles) + } + if _, err := os.Stat(filepath.Join(snapshot.Root, "SHOULD_NEVER_EXIST")); !os.IsNotExist(err) { + t.Fatal("package content executed") + } + if err := packagedigest.Remove(snapshot); err != nil { + t.Fatal(err) + } + archiveAssertEmpty(t, temporary) + }) + } +} + +func TestArchiveLogicalSelectionAndBytePins(t *testing.T) { + var payload []byte + var requests atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requests.Add(1) + _, _ = writer.Write(payload) + })) + defer server.Close() + temporary := t.TempDir() + acquirer := Acquirer{TempRoot: temporary, HTTPClient: server.Client()} + source := sourcespec.Archive{URL: server.URL + "/download", Path: "plugins/demo"} + payload = archiveFixtureZIP(t, archivePackageEntries("repo-old/plugins/demo/"), "old") + first, err := acquirer.AcquireArchive(context.Background(), source, "", "") + if err != nil { + t.Fatal(err) + } + if first.Source.PackageSubpath != "plugins/demo" || first.Source.CanonicalSource != sourcespec.Key(sourcespec.KindArchive, source.URL, "plugins/demo") { + t.Fatalf("physical wrapper leaked into identity: %+v", first.Source) + } + if err := packagedigest.Remove(first); err != nil { + t.Fatal(err) + } + payload = archiveFixtureZIP(t, archivePackageEntries("repo-new/plugins/demo/"), "repacked") + if _, err := acquirer.AcquireArchive(context.Background(), source, first.Source.ArchiveSHA256, first.TreeDigest); err == nil || errors.Is(err, ErrNotArchive) { + t.Fatalf("locked byte change accepted or classified as Git: %v", err) + } + updated, err := acquirer.AcquireArchive(context.Background(), source, "", first.TreeDigest) + if err != nil { + t.Fatal(err) + } + if updated.Source.CanonicalSource != first.Source.CanonicalSource || updated.Source.ArchiveSHA256 == first.Source.ArchiveSHA256 || updated.TreeDigest != first.TreeDigest { + t.Fatalf("wrapper/repack update changed stable ownership: %+v", updated.Source) + } + _ = packagedigest.Remove(updated) + payload = archiveFixtureZIP(t, append(archivePackageEntries("repo/plugins/a/"), archivePackageEntries("repo/plugins/b/")...), "") + before := requests.Load() + _, err = acquirer.AcquireArchive(context.Background(), sourcespec.Archive{URL: source.URL}, "", "") + if err == nil || !strings.Contains(err.Error(), "plugins/a, plugins/b") || errors.Is(err, ErrNotArchive) || requests.Load()-before != 1 { + t.Fatalf("multi-package selection must use one stream: requests=%d, %v", requests.Load()-before, err) + } + archiveAssertEmpty(t, temporary) +} + +func TestArchiveRootManifestNeverFallsBackToNestedPackage(t *testing.T) { + entries := append([]archiveFixtureEntry{{name: "plugin.json", body: "malformed"}}, archivePackageEntries("nested/")...) + snapshot, err := (Acquirer{TempRoot: t.TempDir()}).AcquireArchive(context.Background(), sourcespec.Archive{URL: archiveLocalFile(t, archiveFixtureTAR(t, entries))}, "", "") + if err != nil { + t.Fatal(err) + } + defer func() { _ = packagedigest.Remove(snapshot) }() + if snapshot.Source.PackageSubpath != "" { + t.Fatal("malformed root silently selected nested package") + } + registry, _ := specregistry.New() + if _, err := (loader.Loader{Registry: registry}).LoadSnapshot(context.Background(), snapshot); err == nil { + t.Fatal("malformed root was not rejected by standard loader") + } +} + +func TestArchiveFailuresCleanOwnedFiles(t *testing.T) { + payload := archiveFixtureZIP(t, archivePackageEntries(""), "") + for _, test := range []struct { + name string + source sourcespec.Archive + bytePin, treePin string + }{ + {name: "wrong bytes", bytePin: strings.Repeat("0", 64)}, + {name: "wrong tree", treePin: "sha256:" + strings.Repeat("0", 64)}, + {name: "invalid path", source: sourcespec.Archive{Path: "../escape"}}, + {name: "conflicting byte pins", source: sourcespec.Archive{SHA256: strings.Repeat("1", 64)}, bytePin: strings.Repeat("2", 64)}, + } { + t.Run(test.name, func(t *testing.T) { + temporary := t.TempDir() + test.source.URL = archiveLocalFile(t, payload) + if _, err := (Acquirer{TempRoot: temporary}).AcquireArchive(context.Background(), test.source, test.bytePin, test.treePin); err == nil { + t.Fatal("bad archive request succeeded") + } + archiveAssertEmpty(t, temporary) + }) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + temporary := t.TempDir() + _, err := (Acquirer{TempRoot: temporary}).AcquireArchive(ctx, sourcespec.Archive{URL: archiveLocalFile(t, payload)}, "", "") + if err == nil { + t.Fatal("canceled acquisition succeeded") + } + archiveAssertEmpty(t, temporary) +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_zip.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_zip.go new file mode 100644 index 000000000..0f8da4a2a --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_zip.go @@ -0,0 +1,260 @@ +package sourceacquisition + +import ( + "archive/zip" + "bytes" + "context" + "encoding/binary" + "errors" + "io" + "os" + "sort" +) + +var archiveZIPOrder = binary.LittleEndian + +// archiveValidateZIP supplements archive/zip with a strict single-stream +// boundary audit. archive/zip intentionally accepts SFX prefixes and trailers; +// neither is part of the package archive contract. +func archiveValidateZIP(ctx context.Context, file *os.File, size int64) (*zip.Reader, error) { + count, directoryOffset, directorySize, err := archiveZIPDirectory(file, size) + if err != nil { + return nil, err + } + reader, err := zip.NewReader(file, size) + if err != nil || len(reader.File) != count { + return nil, errors.New("invalid ZIP directory") + } + regions := make([]archiveZIPRegion, 0, count) + offset := directoryOffset + for _, entry := range reader.File { + if err := ctx.Err(); err != nil { + return nil, err + } + record, err := archiveZIPCentral(file, offset, directoryOffset+directorySize, entry) + if err != nil { + return nil, err + } + region, err := archiveZIPLocal(file, record, directoryOffset, entry) + if err != nil { + return nil, err + } + regions = append(regions, region) + offset = record.next + } + if offset != directoryOffset+directorySize { + return nil, errors.New("ZIP has extra central directory records") + } + if err := archiveZIPRegions(regions, directoryOffset); err != nil { + return nil, err + } + return reader, nil +} + +func archiveZIPDirectory(file *os.File, size int64) (int, int64, int64, error) { + if size < 22 || size > archiveMaxBytes { + return 0, 0, 0, errors.New("invalid ZIP stream size") + } + tailSize := min(size, 22+65535) + tail := make([]byte, tailSize) + if _, err := file.ReadAt(tail, size-tailSize); err != nil { + return 0, 0, 0, errors.New("read ZIP end record") + } + end := -1 + for index := len(tail) - 22; index >= 0; index-- { + if string(tail[index:index+4]) == "PK\x05\x06" && index+22+int(archiveZIPOrder.Uint16(tail[index+20:index+22])) == len(tail) { + end = index + break + } + } + if end < 0 { + return 0, 0, 0, errors.New("ZIP requires an exact end record without trailing bytes") + } + eocd := tail[end:] + count := int(archiveZIPOrder.Uint16(eocd[10:12])) + directorySize := int64(archiveZIPOrder.Uint32(eocd[12:16])) + directoryOffset := int64(archiveZIPOrder.Uint32(eocd[16:20])) + if count == 65535 || directorySize == 0xffffffff || directoryOffset == 0xffffffff { + return 0, 0, 0, errors.New("ZIP64 archives exceed the supported ZIP format limits") + } + if archiveZIPOrder.Uint16(eocd[4:6]) != 0 || archiveZIPOrder.Uint16(eocd[6:8]) != 0 || int(archiveZIPOrder.Uint16(eocd[8:10])) != count { + return 0, 0, 0, errors.New("multi-volume ZIP archives are unsupported") + } + if count > 10000 || directoryOffset+directorySize != size-tailSize+int64(end) { + return 0, 0, 0, errors.New("ZIP central directory has invalid bounds or too many entries") + } + return count, directoryOffset, directorySize, nil +} + +type archiveZIPRegion struct{ start, end int64 } +type archiveZIPRecord struct { + localOffset, next int64 + compressed, uncompressed uint32 + flags, method uint16 + name []byte +} + +func archiveZIPCentral(file *os.File, offset, limit int64, entry *zip.File) (archiveZIPRecord, error) { + var empty archiveZIPRecord + var header [46]byte + if offset+46 > limit { + return empty, errors.New("truncated ZIP directory") + } + if _, err := file.ReadAt(header[:], offset); err != nil || string(header[:4]) != "PK\x01\x02" { + return empty, errors.New("invalid ZIP central header") + } + record := archiveZIPRecord{ + flags: archiveZIPOrder.Uint16(header[8:10]), method: archiveZIPOrder.Uint16(header[10:12]), + compressed: archiveZIPOrder.Uint32(header[20:24]), uncompressed: archiveZIPOrder.Uint32(header[24:28]), + localOffset: int64(archiveZIPOrder.Uint32(header[42:46])), + } + nameLength := int64(archiveZIPOrder.Uint16(header[28:30])) + extraLength := int64(archiveZIPOrder.Uint16(header[30:32])) + commentLength := int64(archiveZIPOrder.Uint16(header[32:34])) + record.next = offset + 46 + nameLength + extraLength + commentLength + if record.next > limit || archiveZIPOrder.Uint16(header[34:36]) != 0 { + return empty, errors.New("invalid ZIP entry bounds or disk") + } + if err := archiveZIPRecordLimits(record, entry); err != nil { + return empty, err + } + body := make([]byte, nameLength+extraLength) + if _, err := file.ReadAt(body, offset+46); err != nil || string(body[:nameLength]) != entry.Name { + return empty, errors.New("invalid ZIP entry name") + } + if err := archiveZIPExtra(body[nameLength:]); err != nil { + return empty, err + } + record.name = body[:nameLength] + return record, nil +} + +func archiveZIPRecordLimits(record archiveZIPRecord, entry *zip.File) error { + if record.compressed == 0xffffffff || record.uncompressed == 0xffffffff || record.localOffset == 0xffffffff || entry.ReaderVersion >= 45 { + return errors.New("ZIP64 archives exceed the supported ZIP format limits") + } + if record.flags&^uint16(0x080e) != 0 || record.method != zip.Store && record.method != zip.Deflate || record.method == zip.Store && record.flags&6 != 0 { + return errors.New("ZIP supports only unencrypted store/deflate entries") + } + if record.uncompressed > 64<<20 || int64(record.compressed) > archiveMaxBytes || entry.UncompressedSize64 != uint64(record.uncompressed) || entry.CompressedSize64 != uint64(record.compressed) { + return errors.New("ZIP entry exceeds file byte limits") + } + return nil +} + +func archiveZIPLocal(file *os.File, record archiveZIPRecord, directoryOffset int64, entry *zip.File) (archiveZIPRegion, error) { + var empty archiveZIPRegion + var local [30]byte + if record.localOffset+30 > directoryOffset { + return empty, errors.New("invalid ZIP local bounds") + } + if _, err := file.ReadAt(local[:], record.localOffset); err != nil || string(local[:4]) != "PK\x03\x04" { + return empty, errors.New("ZIP requires local entries without a prefix") + } + nameLength := int64(archiveZIPOrder.Uint16(local[26:28])) + extraLength := int64(archiveZIPOrder.Uint16(local[28:30])) + dataOffset := record.localOffset + 30 + nameLength + extraLength + if archiveZIPOrder.Uint16(local[4:6]) >= 45 || archiveZIPOrder.Uint16(local[6:8]) != record.flags || archiveZIPOrder.Uint16(local[8:10]) != record.method || dataOffset+int64(record.compressed) > directoryOffset || nameLength != int64(len(record.name)) { + return empty, errors.New("inconsistent ZIP local header") + } + body := make([]byte, nameLength+extraLength) + if _, err := file.ReadAt(body, record.localOffset+30); err != nil || !bytes.Equal(body[:nameLength], record.name) { + return empty, errors.New("inconsistent ZIP local name") + } + if err := archiveZIPExtra(body[nameLength:]); err != nil { + return empty, err + } + regionEnd := dataOffset + int64(record.compressed) + if record.flags&8 != 0 { + descriptorSize, err := archiveZIPDescriptor(file, regionEnd, directoryOffset, entry) + if err != nil { + return empty, err + } + regionEnd += descriptorSize + } else if archiveZIPOrder.Uint32(local[14:18]) != entry.CRC32 || archiveZIPOrder.Uint32(local[18:22]) != record.compressed || archiveZIPOrder.Uint32(local[22:26]) != record.uncompressed { + return empty, errors.New("inconsistent ZIP local size or checksum") + } + return archiveZIPRegion{record.localOffset, regionEnd}, nil +} + +func archiveZIPDescriptor(file *os.File, offset, limit int64, entry *zip.File) (int64, error) { + var descriptor [16]byte + if offset+12 > limit { + return 0, errors.New("truncated ZIP data descriptor") + } + n, err := file.ReadAt(descriptor[:], offset) + if err != nil && err != io.EOF || n < 12 { + return 0, errors.New("truncated ZIP data descriptor") + } + values := descriptor[:12] + size := int64(12) + if string(descriptor[:4]) == "PK\x07\x08" { + values, size = descriptor[4:], 16 + } + if offset+size > limit || archiveZIPOrder.Uint32(values[:4]) != entry.CRC32 || uint64(archiveZIPOrder.Uint32(values[4:8])) != entry.CompressedSize64 || uint64(archiveZIPOrder.Uint32(values[8:12])) != entry.UncompressedSize64 { + return 0, errors.New("inconsistent ZIP data descriptor") + } + return size, nil +} + +func archiveZIPRegions(regions []archiveZIPRegion, directoryOffset int64) error { + sort.Slice(regions, func(i, j int) bool { return regions[i].start < regions[j].start }) + var position int64 + for _, region := range regions { + if region.start != position { + return errors.New("ZIP has a prefix, gap, duplicate or overlapping local entry") + } + position = region.end + } + if position != directoryOffset { + return errors.New("ZIP has undeclared data before its directory") + } + return nil +} + +func archiveZIPExtra(body []byte) error { + for len(body) > 0 { + if len(body) < 4 { + return errors.New("invalid ZIP extra record") + } + kind := archiveZIPOrder.Uint16(body[:2]) + size := int(archiveZIPOrder.Uint16(body[2:4])) + if kind == 1 { + return errors.New("ZIP64 records exceed the supported ZIP format limits") + } + if size > len(body)-4 { + return errors.New("truncated ZIP extra record") + } + body = body[4+size:] + } + return nil +} + +func archiveExtractZIP(ctx context.Context, file *os.File, size int64, output *archiveExtraction) error { + reader, err := archiveValidateZIP(ctx, file, size) + if err != nil { + return err + } + for _, entry := range reader.File { + if entry.UncompressedSize64 > 64<<20 { + return errors.New("ZIP entry exceeds file byte limit") + } + mode := entry.Mode() + if !mode.IsRegular() && !mode.IsDir() { + return errors.New("ZIP refuses links and special entries") + } + body, err := entry.Open() + if err != nil { + return errors.New("open ZIP entry") + } + err = output.write(ctx, entry.Name, mode.IsDir(), int64(entry.UncompressedSize64), mode.Perm()&0o111 != 0, body) + closeErr := body.Close() + if err != nil { + return err + } + if closeErr != nil { + return errors.New("close ZIP entry") + } + } + return nil +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go index 5d3b1cdd2..1db3299dd 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go @@ -23,7 +23,7 @@ func (acquirer Acquirer) gitEndpoint(source sourcespec.Git) (string, string, err return "", "", err } if source.Repository != "" && !strings.EqualFold(repository, source.Repository) { - return "", "", errors.New("Git source repository disagrees with its endpoint") + return "", "", errors.New("git source repository disagrees with its endpoint") } return normalized, repository, nil } @@ -124,7 +124,7 @@ func (acquirer Acquirer) newDirectWorkspace(ctx context.Context) (directGitWorks return workspace, nil } func (workspace directGitWorkspace) run(ctx context.Context, native bool, args ...string) ([]byte, error) { - return workspace.acquirer.runner().Run(ctx, Command{Dir: workspace.root, Args: args, NativeCredentials: native, MaxOutputBytes: maxGitHubDiscoveryMetadataBytes}) + return workspace.acquirer.runner().Run(ctx, Command{Dir: workspace.root, Args: args, NativeCredentials: native, mappedLocal: native && workspace.acquirer.mappedLocalArgs(args), MaxOutputBytes: maxGitHubDiscoveryMetadataBytes}) } func (workspace directGitWorkspace) remote(ctx context.Context, args ...string) ([]byte, error) { return workspace.run(ctx, true, append([]string{"-C", workspace.repository}, args...)...) @@ -195,7 +195,7 @@ func (workspace directGitWorkspace) commit(ctx context.Context, endpoint string, } output, err := workspace.run(ctx, false, "-C", workspace.repository, "rev-parse", "--verify", "FETCH_HEAD^{commit}") if err != nil || !sourcespec.ValidCommit(strings.TrimSpace(string(output))) { - return "", errors.New("Git tag did not resolve to a commit") + return "", errors.New("git tag did not resolve to a commit") } return strings.TrimSpace(string(output)), nil } @@ -259,3 +259,20 @@ func directRefError(ctx context.Context) error { } return errors.New("resolve direct Git ref failed; verify native Git credentials and repository access") } + +// URLForRepo is an explicit dependency-injection seam used by local Git fixtures; +// accepted user endpoints still pass strict HTTPS/SSH parsing before mapping. +func (acquirer Acquirer) mappedLocalEndpoint(endpoint string) bool { + return acquirer.URLForRepo != nil && (filepath.IsAbs(endpoint) || strings.HasPrefix(endpoint, "file://")) +} +func (acquirer Acquirer) mappedLocalArgs(args []string) bool { + for index, argument := range args { + if index > 0 && args[index-1] == "-C" { + continue + } + if acquirer.mappedLocalEndpoint(argument) { + return true + } + } + return false +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_policy_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_policy_test.go new file mode 100644 index 000000000..1f476dc13 --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_policy_test.go @@ -0,0 +1,84 @@ +package sourceacquisition + +import ( + "encoding/pem" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +func TestNativeGitRejectsConfiguredFileRewrite(t *testing.T) { + root, _, commit := nativeFixtureRepository(t) + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", home) + // Scoped protocol config is stronger than protocol.allow. GIT_ALLOW_PROTOCOL + // must independently forbid this rewrite of a syntactically HTTPS endpoint. + config := fmt.Sprintf("[protocol \"file\"]\n allow = always\n[url \"file://%s/\"]\n insteadOf = https://fixture.invalid/\n", filepath.ToSlash(root)) + sshFixtureWrite(t, filepath.Join(home, ".gitconfig"), config) + source := sourcespec.Git{URL: "https://fixture.invalid/repo.git"} + if _, err := (Acquirer{TempRoot: t.TempDir()}).AcquireGit(t.Context(), source, commit, ""); err == nil { + t.Fatal("HTTPS endpoint was rewritten to forbidden local file transport") + } +} + +func TestNativeGitRequiresTLSDespiteScopedUserConfig(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { requests.Add(1); w.WriteHeader(http.StatusNotFound) })) + defer server.Close() + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", home) + t.Setenv("GIT_SSL_CAINFO", "") + config := "[http \"" + server.URL + "\"]\n sslVerify = false\n" + sshFixtureWrite(t, filepath.Join(home, ".gitconfig"), config) + _, err := (Acquirer{TempRoot: t.TempDir()}).ResolveGit(t.Context(), sourcespec.Git{URL: server.URL + "/repo.git"}, "default", false) + if err == nil || requests.Load() != 0 { + t.Fatalf("untrusted TLS source reached HTTP despite authoritative verification: requests=%d err=%v", requests.Load(), err) + } +} + +func TestNativeGitRequiresTLSAfterConfiguredHTTPSRewrite(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { requests.Add(1); w.WriteHeader(http.StatusNotFound) })) + defer server.Close() + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", home) + t.Setenv("GIT_SSL_CAINFO", "") + config := "[http \"" + server.URL + "\"]\n sslVerify = false\n[url \"" + server.URL + "/\"]\n insteadOf = https://fixture.invalid/\n" + sshFixtureWrite(t, filepath.Join(home, ".gitconfig"), config) + _, err := (Acquirer{TempRoot: t.TempDir()}).ResolveGit(t.Context(), sourcespec.Git{URL: "https://fixture.invalid/repo.git"}, "default", false) + if err == nil || requests.Load() != 0 { + t.Fatalf("rewritten endpoint bypassed TLS policy: requests=%d err=%v", requests.Load(), err) + } +} + +func TestNativeGitRejectsHTTPSDowngradeRedirect(t *testing.T) { + var downgraded atomic.Int32 + insecure := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { downgraded.Add(1); w.WriteHeader(http.StatusNotFound) })) + defer insecure.Close() + secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, insecure.URL+"/repo.git", http.StatusFound) + })) + defer secure.Close() + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", home) + ca := filepath.Join(home, "ca.pem") + // Reuse the TLS fixture's actual trust anchor without disabling verification. + if err := os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: secure.Certificate().Raw}), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("GIT_SSL_CAINFO", ca) + _, err := (Acquirer{TempRoot: t.TempDir()}).ResolveGit(t.Context(), sourcespec.Git{URL: secure.URL + "/repo.git"}, "default", false) + if err == nil || downgraded.Load() != 0 { + t.Fatalf("Git followed an HTTPS downgrade: requests=%d err=%v", downgraded.Load(), err) + } +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go index d7eced7fd..cc809e10f 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go @@ -1,7 +1,7 @@ package sourceacquisition import ( - "bytes" + "context" "fmt" "net" "os" @@ -69,7 +69,7 @@ func newNativeSSHFixture(t *testing.T, sshd, repositoryRoot, username string) na dir := t.TempDir() sshFixtureKey(t, filepath.Join(dir, "host")) sshFixtureKey(t, filepath.Join(dir, "identity")) - listener, err := net.Listen("tcp", "127.0.0.1:0") + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } @@ -90,15 +90,19 @@ func newNativeSSHFixture(t *testing.T, sshd, repositoryRoot, username string) na config := fmt.Sprintf("Port %d\nListenAddress 127.0.0.1\nHostKey %s\nPidFile %s\nAuthorizedKeysFile %s\nStrictModes no\nPasswordAuthentication no\nKbdInteractiveAuthentication no\nUsePAM no\nPermitRootLogin forced-commands-only\nAllowUsers %s\nLogLevel ERROR\n", port, filepath.Join(dir, "host"), filepath.Join(dir, "pid"), filepath.Join(dir, "authorized_keys"), username) serverConfig := filepath.Join(dir, "sshd_config") sshFixtureWrite(t, serverConfig, config) - var logs bytes.Buffer - command := exec.Command(sshd, "-D", "-e", "-f", serverConfig) - command.Stdout = &logs - command.Stderr = &logs + logs, err := os.Create(filepath.Join(dir, "sshd.log")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = logs.Close() }) + command := exec.CommandContext(t.Context(), sshd, "-D", "-e", "-f", serverConfig) + command.Stdout = logs + command.Stderr = logs if err := command.Start(); err != nil { t.Fatal(err) } t.Cleanup(func() { _ = command.Process.Kill(); _ = command.Wait() }) - waitSSHFixture(t, port, &logs) + waitSSHFixture(t, port, logs.Name()) hostKey, err := os.ReadFile(filepath.Join(dir, "host.pub")) if err != nil { t.Fatal(err) @@ -124,16 +128,17 @@ func sshFixtureWrite(t *testing.T, path, value string) { t.Fatal(err) } } -func waitSSHFixture(t *testing.T, port int, logs *bytes.Buffer) { +func waitSSHFixture(t *testing.T, port int, logs string) { t.Helper() deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { - connection, err := net.DialTimeout("tcp", fmt.Sprintf("127.0.0.1:%d", port), 100*time.Millisecond) + connection, err := (&net.Dialer{Timeout: 100 * time.Millisecond}).DialContext(context.Background(), "tcp", fmt.Sprintf("127.0.0.1:%d", port)) if err == nil { _ = connection.Close() return } time.Sleep(20 * time.Millisecond) } - t.Fatalf("disposable sshd did not start: %s", logs.String()) + body, _ := os.ReadFile(logs) + t.Fatalf("disposable sshd did not start: %s", body) } diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport.go index e2814c276..6b9ef529a 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport.go @@ -2,6 +2,7 @@ package sourceacquisition import ( "context" + "errors" "path/filepath" "strings" "time" @@ -30,7 +31,7 @@ func nativeGitEnvironment(environ []string) []string { } clean = append(clean, item) } - return append(clean, "GIT_TERMINAL_PROMPT=0", "GIT_LFS_SKIP_SMUDGE=1", "GCM_INTERACTIVE=Never", "SSH_ASKPASS_REQUIRE=never") + return append(clean, "GIT_TERMINAL_PROMPT=0", "GIT_LFS_SKIP_SMUDGE=1", "GCM_INTERACTIVE=Never", "SSH_ASKPASS_REQUIRE=never", "GIT_ALLOW_PROTOCOL=https:ssh") } func nativeGitOptions(workspace string) []string { @@ -91,3 +92,30 @@ func ctxOrError(ctx context.Context, err error) error { } return err } + +// URL-specific Git configuration outranks http.sslVerify. Override every +// configured spelling at its own specificity, using the same private repository +// context as the fetch. Values (which may contain credentials) are never read. +// Trusted user configuration is assumed stable during this native command. +func nativeVerificationOptions(ctx context.Context, environ []string, command Command) ([]string, error) { + options := nativeGitOptions(command.Dir) + args := []string{"git"} + if len(command.Args) >= 2 && command.Args[0] == "-C" { + args = append(args, command.Args[:2]...) + } + args = append(args, "config", "--null", "--name-only", "--get-regexp", `^http(\..*)?\.sslverify$`) + result, err := (processadapter.OS{}).RunWithTreeExitGrace(ctx, legacyports.Command{Argv: args, Dir: command.Dir, Env: environ, StdoutLimitBytes: 1 << 20}, time.Second) + if err != nil || (result.ExitCode != 0 && result.ExitCode != 1) { + return nil, errors.New("cannot enforce native Git TLS verification") + } + for _, key := range strings.Split(string(result.Stdout), "\x00") { + if key == "" { + continue + } + if len(key) > 8192 || strings.ContainsAny(key, "\r\n") { + return nil, errors.New("native Git TLS configuration key is invalid") + } + options = append(options, "-c", key+"=true") + } + return options, nil +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport_test.go index 1c8022c33..1e259d7f9 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport_test.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport_test.go @@ -20,12 +20,12 @@ import ( func TestNativeGitEnvironmentSeparatesAuthenticationFromRouting(t *testing.T) { env := nativeGitEnvironment([]string{"HOME=/user", "SSH_AUTH_SOCK=/agent", "GIT_ASKPASS=/helper", "GIT_SSH_COMMAND=ssh -F /config", "GIT_SSL_CAINFO=/ca", "GIT_DIR=/evil", "GIT_WORK_TREE=/evil", "GIT_CONFIG_COUNT=1", "GIT_CONFIG_KEY_0=http.extraHeader", "GIT_CONFIG_VALUE_0=secret", "GIT_TRACE=/leak", "GIT_SSL_NO_VERIFY=1"}) joined := strings.Join(env, "\n") - for _, wanted := range []string{"HOME=/user", "SSH_AUTH_SOCK=/agent", "GIT_ASKPASS=/helper", "GIT_SSH_COMMAND=ssh -F /config", "GIT_SSL_CAINFO=/ca", "GIT_TERMINAL_PROMPT=0", "GCM_INTERACTIVE=Never"} { + for _, wanted := range []string{"HOME=/user", "SSH_AUTH_SOCK=/agent", "GIT_ASKPASS=/helper", "GIT_SSH_COMMAND=ssh -F /config", "GIT_SSL_CAINFO=/ca", "GIT_TERMINAL_PROMPT=0", "GCM_INTERACTIVE=Never", "GIT_ALLOW_PROTOCOL=https:ssh"} { if !strings.Contains(joined, wanted) { t.Fatalf("missing native auth input %s", wanted) } } - for _, forbidden := range []string{"GIT_DIR=", "GIT_WORK_TREE=", "GIT_CONFIG_", "GIT_TRACE=", "GIT_SSL_NO_VERIFY="} { + for _, forbidden := range []string{"GIT_DIR=", "GIT_WORK_TREE=", "GIT_CONFIG_", "GIT_TRACE=", "GIT_SSL_NO_VERIFY=1"} { if strings.Contains(joined, forbidden) { t.Fatalf("unsafe native environment %s", forbidden) } diff --git a/install/integrationctl/agentplugins/environment/sources.go b/install/integrationctl/agentplugins/environment/sources.go index 52d79fd1c..c16f53ae2 100644 --- a/install/integrationctl/agentplugins/environment/sources.go +++ b/install/integrationctl/agentplugins/environment/sources.go @@ -2,7 +2,6 @@ package environment import ( "errors" - "strings" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" ) @@ -27,6 +26,9 @@ func (source GitSource) ValidateEndpoint() error { } return nil } + if _, err := sourcespec.NormalizeSubpath(source.Path); err != nil { + return err + } endpoint, repository, err := sourcespec.NormalizeGitURL(source.URL) if err != nil || endpoint != source.URL || repository != "" { return errors.New("git URL must be a normalized HTTPS/SSH endpoint; use repository for canonical GitHub HTTPS") @@ -102,5 +104,5 @@ func (requirement Requirement) SameSource(other Requirement) bool { return false } b, err := other.SourceKey() - return err == nil && strings.Compare(a, b) == 0 + return err == nil && a == b } diff --git a/install/integrationctl/agentplugins/environment/sources_test.go b/install/integrationctl/agentplugins/environment/sources_test.go index 387014493..113251a73 100644 --- a/install/integrationctl/agentplugins/environment/sources_test.go +++ b/install/integrationctl/agentplugins/environment/sources_test.go @@ -46,13 +46,13 @@ func TestRemoteSourceLockIdentityAndOldReaderRefusal(t *testing.T) { } changed := lock.Packages[0] if changed.Source.Git != nil { - copy := *changed.Source.Git - copy.URL = "ssh://git@other.test/Team/Repo" - changed.Source.Git = © + cloned := *changed.Source.Git + cloned.URL = "ssh://git@other.test/Team/Repo" + changed.Source.Git = &cloned } else { - copy := *changed.Source.Archive - copy.SHA256 = "bad" - changed.Source.Archive = © + cloned := *changed.Source.Archive + cloned.SHA256 = "bad" + changed.Source.Archive = &cloned } lock.Packages[0] = changed if lock.Validate(reqs) == nil { @@ -119,3 +119,17 @@ func TestArchiveMovingUpdateAndSyncRevision(t *testing.T) { t.Fatal("explicit checksum updated without intent change") } } + +func TestNewURLGitRequiresSharedPortablePathGrammar(t *testing.T) { + for _, packagePath := range []string{"CON", "a\x01b", "a/../b", "a\\b"} { + requirement := Requirement{Alias: "demo", Git: &GitSource{URL: "https://example.test/repo.git", Path: packagePath, Selector: Selector{Kind: SelectorDefault}}} + if requirement.Validate() == nil { + t.Fatalf("new URL Git accepted nonportable package path %q", packagePath) + } + } + // Existing repository intent uses the historical path admission contract. + old := Requirement{Alias: "demo", Git: &GitSource{Repository: "acme/repo", Path: "CON", Selector: Selector{Kind: SelectorDefault}}} + if err := old.Validate(); err != nil { + t.Fatalf("historical repository path contract changed: %v", err) + } +} diff --git a/install/integrationctl/agentplugins/sourcespec/git.go b/install/integrationctl/agentplugins/sourcespec/git.go index 11ab71bcf..8e36780b0 100644 --- a/install/integrationctl/agentplugins/sourcespec/git.go +++ b/install/integrationctl/agentplugins/sourcespec/git.go @@ -4,6 +4,7 @@ import ( "errors" "net/url" "strings" + "unicode" ) // NormalizeGitURL accepts only native HTTPS/SSH transports, never local paths, @@ -14,36 +15,9 @@ func NormalizeGitURL(value string) (string, string, error) { return "", "", errors.New("unsafe Git endpoint") } if !strings.Contains(value, "://") { - left, right, ok := strings.Cut(value, ":") - if !ok || right == "" || strings.HasPrefix(right, "-") || strings.ContainsAny(left, " /?#") || strings.ContainsAny(right, "?#") || (!strings.Contains(left, "@") && !strings.Contains(left, ".")) { - return "", "", errors.New("Git endpoint must use HTTPS, ssh:// or user@host:path") - } - user, host, hasUser := strings.Cut(left, "@") - if hasUser { - if user == "" || host == "" || strings.Contains(host, "@") { - return "", "", errors.New("invalid SSH endpoint") - } - } else { - host = user - user = "" - } - if strings.ContainsAny(host, "[]: ") || strings.ContainsAny(user, ": ") { - return "", "", errors.New("invalid SSH endpoint") - } - for index, segment := range strings.Split(right, "/") { - if index == 0 && segment == "" && strings.HasPrefix(right, "/") { - continue - } - if segment == "." || segment == ".." || segment == "" { - return "", "", errors.New("unsafe SSH repository path") - } - } - prefix := strings.ToLower(host) - if hasUser { - prefix = user + "@" + prefix - } - return prefix + ":" + right, "", nil + return normalizeSCP(value) } + u, err := url.Parse(value) if err != nil || (u.Scheme != "https" && u.Scheme != "ssh") { return "", "", errors.New("Git endpoint must use HTTPS or SSH") @@ -52,17 +26,8 @@ func NormalizeGitURL(value string) (string, string, error) { if err != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { return "", "", errors.New("Git endpoint cannot contain query or fragment") } - if u.User != nil { - _, password := u.User.Password() - if password || u.User.Username() == "" || strings.ContainsAny(u.User.Username(), "@:/ ") { - return "", "", errors.New("Git endpoint cannot contain credentials") - } - if u.Scheme == "https" { - if !isAzureHost(u.Hostname()) { - return "", "", errors.New("HTTPS Git endpoint cannot contain user information") - } - u.User = nil - } + if err := normalizeGitUser(u); err != nil { + return "", "", err } repository := "" if u.Scheme == "https" && u.Host == "github.com" { @@ -79,6 +44,61 @@ func NormalizeGitURL(value string) (string, string, error) { return u.String(), repository, nil } +func normalizeGitUser(u *url.URL) error { + if u.User != nil { + _, password := u.User.Password() + if password || u.User.Username() == "" || strings.ContainsAny(u.User.Username(), "@:/ ") || strings.ContainsFunc(u.User.Username(), unicode.IsControl) { + return errors.New("Git endpoint cannot contain credentials") + } + if u.Scheme == "https" { + if !isAzureHost(u.Hostname()) { + return errors.New("HTTPS Git endpoint cannot contain user information") + } + u.User = nil + } + } + return nil +} + +func normalizeSCP(value string) (string, string, error) { + left, right, ok := strings.Cut(value, ":") + if !ok || right == "" || strings.HasPrefix(right, "-") || strings.ContainsAny(left, " /?#") || strings.ContainsAny(right, "?#") || (!strings.Contains(left, "@") && !strings.Contains(left, ".")) { + return "", "", errors.New("Git endpoint must use HTTPS, ssh:// or user@host:path") + } + user, host, hasUser := strings.Cut(left, "@") + if hasUser { + if user == "" || host == "" || strings.Contains(host, "@") { + return "", "", errors.New("invalid SSH endpoint") + } + } else { + host = user + user = "" + } + if strings.ContainsAny(host, "[]: ") || strings.ContainsAny(user, ": ") { + return "", "", errors.New("invalid SSH endpoint") + } + if err := validateSCPPath(right); err != nil { + return "", "", err + } + prefix := strings.ToLower(host) + if hasUser { + prefix = user + "@" + prefix + } + return prefix + ":" + right, "", nil +} + +func validateSCPPath(right string) error { + for index, segment := range strings.Split(right, "/") { + if index == 0 && segment == "" && strings.HasPrefix(right, "/") { + continue + } + if segment == "." || segment == ".." || segment == "" { + return errors.New("unsafe SSH repository path") + } + } + return nil +} + func isAzureHost(host string) bool { return host == "dev.azure.com" || strings.HasSuffix(host, ".visualstudio.com") } diff --git a/install/integrationctl/agentplugins/sourcespec/parse.go b/install/integrationctl/agentplugins/sourcespec/parse.go index c304d9186..509708ae1 100644 --- a/install/integrationctl/agentplugins/sourcespec/parse.go +++ b/install/integrationctl/agentplugins/sourcespec/parse.go @@ -13,12 +13,16 @@ func Parse(raw string) (Spec, error) { return Spec{}, errors.New("invalid source input") } out := Spec{Requested: raw} + if !strings.Contains(raw, "://") && !strings.HasPrefix(raw, "github:") && !looksLikeSCP(raw) && HasArchiveSuffix(raw) { + out.Kind, out.Archive = KindLocalArchive, &Archive{URL: raw} + return out, nil + } if strings.HasPrefix(raw, "github:") || (!strings.Contains(raw, ":") && strings.Contains(raw, "@")) { g, err := parseShorthand(raw) out.Kind, out.Git = KindGit, &g return out, err } - if strings.HasPrefix(raw, "git+https://") || strings.HasPrefix(raw, "ssh://") || (!strings.Contains(raw, "://") && strings.Contains(raw, "@") && strings.Contains(raw, ":")) { + if strings.HasPrefix(raw, "git+https://") || strings.HasPrefix(raw, "ssh://") || looksLikeSCP(raw) { g, err := parseGit(raw) out.Kind, out.Git = KindGit, &g return out, err @@ -31,6 +35,18 @@ func Parse(raw string) (Spec, error) { } return Spec{}, errors.New("not a direct remote or archive source") } + return parseHTTPS(raw, out) +} + +func looksLikeSCP(raw string) bool { + if strings.Contains(raw, "://") { + return false + } + host, _, ok := strings.Cut(raw, ":") + return ok && !strings.ContainsAny(host, "/\\") && (strings.Contains(host, "@") || strings.Contains(host, ".")) +} + +func parseHTTPS(raw string, out Spec) (Spec, error) { u, err := url.Parse(raw) if err != nil { return Spec{}, errors.New("invalid HTTPS source") @@ -62,9 +78,9 @@ func Parse(raw string) (Spec, error) { func parseArchive(u *url.URL) (Archive, error) { fragment := u.Fragment - copy := *u - copy.Fragment = "" - endpoint, err := NormalizeArchiveURL(copy.String()) + base := *u + base.Fragment = "" + endpoint, err := NormalizeArchiveURL(base.String()) if err != nil { return Archive{}, err } diff --git a/install/integrationctl/agentplugins/sourcespec/provider.go b/install/integrationctl/agentplugins/sourcespec/provider.go index 9dec28789..2f1231a3c 100644 --- a/install/integrationctl/agentplugins/sourcespec/provider.go +++ b/install/integrationctl/agentplugins/sourcespec/provider.go @@ -15,100 +15,119 @@ func parseProvider(u *url.URL) (Git, bool, error) { g, err := parseAzure(u) return g, true, err } - segments := strings.Split(strings.Trim(u.Path, "/"), "/") if host == "github.com" { - if len(segments) < 2 { - return Git{}, true, errors.New("GitHub source requires owner/repo") - } - base := *u - base.Path = "/" + strings.Join(segments[:2], "/") - base.RawQuery = "" - base.RawPath = "" - endpoint, repo, err := NormalizeGitURL(base.String()) - if err != nil { - return Git{}, true, err - } - g := Git{URL: endpoint, Repository: repo} - if u.RawQuery != "" || u.ForceQuery { - return Git{}, true, errors.New("GitHub source cannot contain query parameters") - } - if len(segments) == 2 { - return g, true, nil - } - if len(segments) < 4 || segments[2] != "tree" { - return Git{}, true, errors.New("choose a repository or tree package root URL") - } - g.Browser = &BrowserTree{Provider: "github", Suffix: strings.Join(segments[3:], "/")} - return g, true, nil + g, err := parseGitHubBrowser(u) + return g, true, err + } + if strings.Contains(u.Path, "/-/tree/") || host == "gitlab.com" { + g, err := parseGitLabBrowser(u) + return g, true, err + } + return Git{}, false, nil +} + +func parseGitHubBrowser(u *url.URL) (Git, error) { + segments := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(segments) < 2 { + return Git{}, errors.New("GitHub source requires owner/repo") + } + if u.RawQuery != "" || u.ForceQuery { + return Git{}, errors.New("GitHub source cannot contain query parameters") + } + base := *u + base.Path, base.RawPath, base.RawQuery = "/"+strings.Join(segments[:2], "/"), "", "" + endpoint, repo, err := NormalizeGitURL(base.String()) + if err != nil { + return Git{}, err + } + g := Git{URL: endpoint, Repository: repo} + if len(segments) == 2 { + return g, nil + } + if len(segments) < 4 || segments[2] != "tree" { + return Git{}, errors.New("choose a repository or tree package root URL") + } + g.Browser = &BrowserTree{Provider: "github", Suffix: strings.Join(segments[3:], "/")} + return g, nil +} + +func parseGitLabBrowser(u *url.URL) (Git, error) { + if u.RawQuery != "" || u.ForceQuery { + return Git{}, errors.New("GitLab source cannot contain query parameters") } marker := "/-/tree/" + base := *u + var browser *BrowserTree if index := strings.Index(u.Path, marker); index >= 0 { - base := *u base.Path = u.Path[:index] - base.RawPath = "" - base.RawQuery = "" - endpoint, repo, err := NormalizeGitURL(base.String() + ".git") - if err != nil { - return Git{}, true, err - } - if u.RawQuery != "" || u.ForceQuery { - return Git{}, true, errors.New("GitLab tree source cannot contain query parameters") - } - return Git{URL: endpoint, Repository: repo, Browser: &BrowserTree{Provider: "gitlab", Suffix: u.Path[index+len(marker):]}}, true, nil - } - if host == "gitlab.com" { - if len(segments) < 2 || strings.Contains(u.Path, "/-/") { - return Git{}, true, errors.New("choose a GitLab repository or tree package root URL") - } - copy := *u - copy.Path = strings.TrimSuffix(copy.Path, "/") + ".git" - copy.RawPath = "" - endpoint, repo, err := NormalizeGitURL(copy.String()) - return Git{URL: endpoint, Repository: repo}, true, err + browser = &BrowserTree{Provider: "gitlab", Suffix: u.Path[index+len(marker):]} + } else if len(strings.Split(strings.Trim(u.Path, "/"), "/")) < 2 || strings.Contains(u.Path, "/-/") { + return Git{}, errors.New("choose a GitLab repository or tree package root URL") } - return Git{}, false, nil + base.Path = strings.TrimSuffix(base.Path, "/") + ".git" + base.RawPath, base.RawQuery = "", "" + endpoint, repo, err := NormalizeGitURL(base.String()) + return Git{URL: endpoint, Repository: repo, Browser: browser}, err } func parseAzure(u *url.URL) (Git, error) { - segments := strings.Split(strings.Trim(u.Path, "/"), "/") + if err := validateAzurePath(u.Path); err != nil { + return Git{}, err + } + q, err := azureQuery(u.RawQuery) + if err != nil { + return Git{}, err + } + base := *u + base.RawQuery = "" + base.ForceQuery = false + base.Fragment = "" + endpoint, repo, err := NormalizeGitURL(base.String()) + if err != nil { + return Git{}, err + } + g := Git{URL: endpoint, Repository: repo} + g.Path, err = NormalizeSubpath(strings.TrimPrefix(q.Get("path"), "/")) + if err != nil { + return Git{}, err + } + return azureVersion(g, q.Get("version")) +} + +func validateAzurePath(value string) error { + segments := strings.Split(strings.Trim(value, "/"), "/") index := -1 for i, s := range segments { if s == "_git" { if index >= 0 { - return Git{}, errors.New("invalid Azure repository URL") + return errors.New("invalid Azure repository URL") } index = i } } if index < 1 || index+2 != len(segments) { - return Git{}, errors.New("Azure source requires an _git repository URL") + return errors.New("azure source requires an _git repository URL") } - q, err := url.ParseQuery(u.RawQuery) + return nil +} + +func azureQuery(raw string) (url.Values, error) { + q, err := url.ParseQuery(raw) if err != nil { - return Git{}, errors.New("invalid Azure source query") + return nil, errors.New("invalid Azure source query") } for key, values := range q { if len(values) != 1 || (key != "path" && key != "version" && key != "_a") { - return Git{}, errors.New("unsupported Azure source query") + return nil, errors.New("unsupported Azure source query") } } if q.Get("_a") != "" && q.Get("_a") != "contents" { - return Git{}, errors.New("choose Azure repository contents") - } - base := *u - base.RawQuery = "" - base.ForceQuery = false - base.Fragment = "" - endpoint, repo, err := NormalizeGitURL(base.String()) - if err != nil { - return Git{}, err + return nil, errors.New("choose Azure repository contents") } - g := Git{URL: endpoint, Repository: repo} - g.Path, err = NormalizeSubpath(strings.TrimPrefix(q.Get("path"), "/")) - if err != nil { - return Git{}, err - } - version := q.Get("version") + return q, nil +} + +func azureVersion(g Git, version string) (Git, error) { if version != "" { if len(version) < 3 { return Git{}, errors.New("invalid Azure source version") @@ -120,7 +139,7 @@ func parseAzure(u *url.URL) (Git, error) { g.Ref = "refs/tags/" + version[2:] case "GC": if !ValidCommit(version[2:]) { - return Git{}, errors.New("Azure commit requires a full lowercase SHA") + return Git{}, errors.New("azure commit requires a full lowercase SHA") } g.Commit = version[2:] default: diff --git a/install/integrationctl/agentplugins/sourcespec/source_test.go b/install/integrationctl/agentplugins/sourcespec/source_test.go index 3b4ebfd92..606dffdb5 100644 --- a/install/integrationctl/agentplugins/sourcespec/source_test.go +++ b/install/integrationctl/agentplugins/sourcespec/source_test.go @@ -21,6 +21,7 @@ func TestParseDirectSources(t *testing.T) { {"git+https://example.test/Team/Repo", KindGit, &Git{URL: "https://example.test/Team/Repo"}, nil}, {"ssh://git@example.test:2222/Team/Repo#main", KindGit, &Git{URL: "ssh://git@example.test:2222/Team/Repo", Ref: "main"}, nil}, {"git@ssh.dev.azure.com:v3/org/project/repo", KindGit, &Git{URL: "git@ssh.dev.azure.com:v3/org/project/repo"}, nil}, + {"git.example.test:team/repo.git", KindGit, &Git{URL: "git.example.test:team/repo.git"}, nil}, {"git@example.test:/srv/repo.git", KindGit, &Git{URL: "git@example.test:/srv/repo.git"}, nil}, {"https://github.com/owner/repo/tree/feature/a/pkg", KindGit, &Git{URL: "https://github.com/owner/repo.git", Repository: "owner/repo", Browser: &BrowserTree{Provider: "github", Suffix: "feature/a/pkg"}}, nil}, {"https://gitlab.com/group/sub/repo/-/tree/main/pkg", KindGit, &Git{URL: "https://gitlab.com/group/sub/repo.git", Browser: &BrowserTree{Provider: "gitlab", Suffix: "main/pkg"}}, nil}, @@ -33,6 +34,7 @@ func TestParseDirectSources(t *testing.T) { {"https://downloads.test/package#sha256=" + strings.Repeat("b", 64) + "&path=plugins%2Fdemo", KindArchive, nil, &Archive{URL: "https://downloads.test/package", SHA256: strings.Repeat("b", 64), Path: "plugins/demo"}}, {"https://example.test/package", KindHTTPSAuto, nil, &Archive{URL: "https://example.test/package"}}, {"./plugin.tgz", KindLocalArchive, nil, &Archive{URL: "./plugin.tgz"}}, + {"./package@1.0.zip", KindLocalArchive, nil, &Archive{URL: "./package@1.0.zip"}}, {"plugin.zip", KindLocalArchive, nil, &Archive{URL: "plugin.zip"}}, } for _, tt := range tests { @@ -50,6 +52,8 @@ func TestParseDirectSources(t *testing.T) { func TestUnsafeSourcesFailWithoutEchoingInput(t *testing.T) { for _, raw := range []string{ + "ssh://user%0Aname@example.test/repo", "ssh://user%01name@example.test/repo", + "C:/repo.git", "C:\\repo.git", "https://secret:TOKEN@example.test/r.git", "ssh://secret:TOKEN@example.test/r.git", "https://example.test/a.zip?token=TOKEN", "https://example.test/r.git#--upload-pack=TOKEN", "https://example.test/r.git#main//../escape", "git+file:///tmp/repo", From 8bc6adefe880228bd2ccd4440a481bc574bbd865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Fri, 23 Oct 2026 19:16:54 +0300 Subject: [PATCH 06/25] feat(sources): integrate direct transports with saved package lifecycle --- cli/cmd/agentplugins/main.go | 32 ++ cli/internal/agentpluginscli/add_multi.go | 5 +- .../agentpluginscli/deps_candidates.go | 4 +- cli/internal/agentpluginscli/deps_commands.go | 3 + cli/internal/agentpluginscli/deps_e2e_test.go | 10 + .../deps_fresh_restore_e2e_test.go | 10 + cli/internal/agentpluginscli/deps_replay.go | 7 +- cli/internal/agentpluginscli/deps_resolver.go | 97 ++++--- cli/internal/agentpluginscli/deps_source.go | 104 +++++++ cli/internal/agentpluginscli/deps_staging.go | 15 +- cli/internal/agentpluginscli/deps_status.go | 8 +- cli/internal/agentpluginscli/deps_sync.go | 29 +- cli/internal/agentpluginscli/lifecycle.go | 3 + cli/internal/agentpluginscli/saved_add.go | 18 +- .../agentpluginscli/saved_add_e2e_test.go | 14 + cli/internal/agentpluginscli/saved_direct.go | 142 +++++++++ .../agentpluginscli/saved_lifecycle.go | 11 +- cli/internal/agentpluginscli/source.go | 14 +- cli/internal/agentpluginscli/source_direct.go | 135 +++++++++ .../source_identity_e2e_test.go | 145 ++++++++++ .../source_transports_e2e_test.go | 273 ++++++++++++++++++ .../archive-code-review-r1.md | 30 ++ .../archive-code-review-r2.md | 26 ++ .../adapters/sourceacquisition/archive.go | 3 + .../sourceacquisition/archive_download.go | 6 +- .../adapters/sourceacquisition/archive_tar.go | 5 +- .../agentplugins/adapters/statev2/store.go | 4 + .../agentplugins/domain/identity.go | 2 +- .../agentplugins/installer/acquired_source.go | 61 ++++ .../agentplugins/installer/group_prepare.go | 4 +- .../agentplugins/installer/inspect.go | 4 + .../agentplugins/installer/package.go | 14 +- .../agentplugins/sourcespec/parse.go | 16 +- .../agentplugins/sourcespec/source_test.go | 3 + .../agentplugins/usecase/binding.go | 1 + .../agentplugins/usecase/group_commit.go | 7 +- .../agentplugins/usecase/installation.go | 15 +- 37 files changed, 1156 insertions(+), 124 deletions(-) create mode 100644 cli/internal/agentpluginscli/deps_source.go create mode 100644 cli/internal/agentpluginscli/saved_direct.go create mode 100644 cli/internal/agentpluginscli/source_direct.go create mode 100644 cli/internal/agentpluginscli/source_identity_e2e_test.go create mode 100644 cli/internal/agentpluginscli/source_transports_e2e_test.go create mode 100644 docs/plans/source-transports-e2e/archive-code-review-r1.md create mode 100644 docs/plans/source-transports-e2e/archive-code-review-r2.md create mode 100644 install/integrationctl/agentplugins/installer/acquired_source.go diff --git a/cli/cmd/agentplugins/main.go b/cli/cmd/agentplugins/main.go index f5c28af79..afbfcf652 100644 --- a/cli/cmd/agentplugins/main.go +++ b/cli/cmd/agentplugins/main.go @@ -29,6 +29,7 @@ import ( clientregistry "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/clients/all" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/managedstdio" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" ) var ( @@ -295,3 +296,34 @@ func (acquirer lazySourceAcquirer) ResolveGitHubRef(ctx context.Context, reposit } return acquirer.acquirer.ResolveGitHubRef(ctx, repository, ref) } + +func (acquirer lazySourceAcquirer) ResolveGitSource(ctx context.Context, source sourcespec.Git) (sourceacquisition.ResolvedGit, error) { + if err := acquirer.prepare(); err != nil { + return sourceacquisition.ResolvedGit{}, err + } + return acquirer.acquirer.ResolveGitSource(ctx, source) +} +func (acquirer lazySourceAcquirer) ResolveGit(ctx context.Context, source sourcespec.Git, selector string, exact bool) (sourceacquisition.RefResolution, error) { + if err := acquirer.prepare(); err != nil { + return sourceacquisition.RefResolution{}, err + } + return acquirer.acquirer.ResolveGit(ctx, source, selector, exact) +} +func (acquirer lazySourceAcquirer) DiscoverGitPackages(ctx context.Context, source sourcespec.Git, commit string) ([]string, error) { + if err := acquirer.prepare(); err != nil { + return nil, err + } + return acquirer.acquirer.DiscoverGitPackages(ctx, source, commit) +} +func (acquirer lazySourceAcquirer) AcquireGit(ctx context.Context, source sourcespec.Git, commit, digest string) (domain.PackageSnapshot, error) { + if err := acquirer.prepare(); err != nil { + return domain.PackageSnapshot{}, err + } + return acquirer.acquirer.AcquireGit(ctx, source, commit, digest) +} +func (acquirer lazySourceAcquirer) AcquireArchive(ctx context.Context, source sourcespec.Archive, archiveSHA, tree string) (domain.PackageSnapshot, error) { + if err := acquirer.prepare(); err != nil { + return domain.PackageSnapshot{}, err + } + return acquirer.acquirer.AcquireArchive(ctx, source, archiveSHA, tree) +} diff --git a/cli/internal/agentpluginscli/add_multi.go b/cli/internal/agentpluginscli/add_multi.go index 1fd0dbc06..b2ad800ad 100644 --- a/cli/internal/agentpluginscli/add_multi.go +++ b/cli/internal/agentpluginscli/add_multi.go @@ -438,7 +438,7 @@ func newAddAcquisitionProof(loaded loadedPackage) (addAcquisitionProof, error) { ManifestDigest: loaded.envelope.ManifestDigest, ClosureDigest: groupedAcquisitionClosureDigest(sourceKind, loaded.envelope.Source, loaded.envelope.TreeDigest, loaded.envelope.ManifestDigest), SourceKind: sourceKind, - Fetched: loaded.envelope.Source.Repository != "", + Fetched: loaded.envelope.Source.Repository != "" || loaded.envelope.Source.SourceKind != "", Validated: true, }, nil } @@ -447,6 +447,9 @@ func acquisitionSourceKind(loaded loadedPackage) string { if loaded.origin == domain.OriginModeDirectory { return "directory" } + if loaded.envelope.Source.SourceKind != "" { + return loaded.envelope.Source.SourceKind + } if loaded.envelope.Source.Repository != "" { return "github" } diff --git a/cli/internal/agentpluginscli/deps_candidates.go b/cli/internal/agentpluginscli/deps_candidates.go index 347a642c9..058622fcf 100644 --- a/cli/internal/agentpluginscli/deps_candidates.go +++ b/cli/internal/agentpluginscli/deps_candidates.go @@ -95,7 +95,7 @@ func (resolver dependencyResolver) AcquireCandidate(ctx context.Context, require return environment.ResolvedPackage{}, err } result := environment.ResolvedPackage{Package: pkg, Dependencies: facts} - return result, resolver.finishLoaded(loaded) + return result, resolver.finishLoaded(loaded, pkg) } func dependencyManifestAlias(name string) string { @@ -119,7 +119,7 @@ func describePinnedDependency(ctx context.Context, app App, pin environment.Lock if err != nil { return environment.ResolvedPackage{}, err } - input := domain.LoadInput{SnapshotRoot: pkg.Root, TreeDigest: pkg.Metadata.TreeDigest, ExecutableFiles: append([]string{}, pkg.Metadata.ExecutableFiles...), Source: pkg.Metadata.Source} + input := domain.LoadInput{SnapshotRoot: pkg.Root, TreeDigest: pkg.Metadata.TreeDigest, ExecutableFiles: append([]string{}, pkg.Metadata.ExecutableFiles...), Source: identityFromLocked(pin)} envelope, err := app.loadStandardEnvelope(ctx, input) if err != nil { return environment.ResolvedPackage{}, err diff --git a/cli/internal/agentpluginscli/deps_commands.go b/cli/internal/agentpluginscli/deps_commands.go index 1d74d4f71..e2ca21e09 100644 --- a/cli/internal/agentpluginscli/deps_commands.go +++ b/cli/internal/agentpluginscli/deps_commands.go @@ -151,6 +151,9 @@ func runDependencyLock(ctx context.Context, app App, opts *options, updateAlias if readErr != nil || string(manifestAfter) != string(manifestBefore) { return dependencyLockResult{}, errors.New("uap.toml changed during lock resolution; rerun deps lock") } + if err := ensureDependencyStore(ctx, app, locked); err != nil { + return dependencyLockResult{}, err + } if err := writeDependencyLock(lockPath, locked); err != nil { return dependencyLockResult{}, err } diff --git a/cli/internal/agentpluginscli/deps_e2e_test.go b/cli/internal/agentpluginscli/deps_e2e_test.go index c1be0c215..09d06cf1f 100644 --- a/cli/internal/agentpluginscli/deps_e2e_test.go +++ b/cli/internal/agentpluginscli/deps_e2e_test.go @@ -18,6 +18,7 @@ import ( "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/installer" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" legacyports "github.com/777genius/plugin-kit-ai/install/integrationctl/ports" ) @@ -48,6 +49,15 @@ func (s *dependencyTestSource) ResolveGitHubSelector(ctx context.Context, repo, return s.Acquirer.ResolveGitHubSelector(ctx, repo, selector) } +func (s *dependencyTestSource) AcquireGit(ctx context.Context, source sourcespec.Git, commit, digest string) (domain.PackageSnapshot, error) { + s.downloads++ + return s.Acquirer.AcquireGit(ctx, source, commit, digest) +} +func (s *dependencyTestSource) ResolveGit(ctx context.Context, source sourcespec.Git, selector string, exact bool) (sourceacquisition.RefResolution, error) { + s.lookups++ + return s.Acquirer.ResolveGit(ctx, source, selector, exact) +} + // Real Git acquisition, standard loader, digest store, public SDK transaction, // Codex projection and CLI commands run against exclusively disposable roots. func TestDependencyEnvironmentCLIEndToEnd(t *testing.T) { diff --git a/cli/internal/agentpluginscli/deps_fresh_restore_e2e_test.go b/cli/internal/agentpluginscli/deps_fresh_restore_e2e_test.go index 9d0f5e083..0d5481bdf 100644 --- a/cli/internal/agentpluginscli/deps_fresh_restore_e2e_test.go +++ b/cli/internal/agentpluginscli/deps_fresh_restore_e2e_test.go @@ -20,6 +20,7 @@ import ( "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/installer" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" legacyports "github.com/777genius/plugin-kit-ai/install/integrationctl/ports" ) @@ -40,6 +41,15 @@ func (source *freshRestoreSource) AcquireGitHubVerified(ctx context.Context, rep return snapshot, err } +func (source *freshRestoreSource) AcquireGit(ctx context.Context, git sourcespec.Git, commit, digest string) (domain.PackageSnapshot, error) { + source.verified = append(source.verified, freshRestoreFetch{git.Repository, commit, git.Path, digest}) + snapshot, err := source.dependencyTestSource.AcquireGit(ctx, git, commit, digest) + if err == nil { + source.snapshots = append(source.snapshots, snapshot.Root) + } + return snapshot, err +} + type freshRestoreRunner struct { *dependencyFixtureRunner calls int diff --git a/cli/internal/agentpluginscli/deps_replay.go b/cli/internal/agentpluginscli/deps_replay.go index 2c20ec459..f45eab4a4 100644 --- a/cli/internal/agentpluginscli/deps_replay.go +++ b/cli/internal/agentpluginscli/deps_replay.go @@ -86,12 +86,7 @@ func verifyDeclaredEdges(parent string, declarations map[string]environment.Requ if err != nil { return err } - var selector environment.Selector - if requirement.Git != nil { - selector = requirement.Git.Selector - } else { - selector = requirement.Registry.Selector - } + selector := requirement.RequestedSelector() expected[key] = selector } for key, pin := range bySource { diff --git a/cli/internal/agentpluginscli/deps_resolver.go b/cli/internal/agentpluginscli/deps_resolver.go index 303a3e31b..486aa230b 100644 --- a/cli/internal/agentpluginscli/deps_resolver.go +++ b/cli/internal/agentpluginscli/deps_resolver.go @@ -11,11 +11,11 @@ import ( "strings" "github.com/777genius/plugin-kit-ai/install/integrationctl/adapters/atomicfile" - "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/directoryv1" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/environmentstore" - "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/sourceacquisition" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" ) type dependencyResolver struct { @@ -78,14 +78,14 @@ func (resolver dependencyResolver) Resolve(ctx context.Context, requirement envi if requirement.Git != nil { return resolver.resolveGit(ctx, requirement, targets, previous, update) } + if requirement.Archive != nil { + return resolver.resolveArchive(ctx, requirement, targets) + } return resolver.resolveRegistry(ctx, requirement, targets) } func sameRequestedSelector(requirement environment.Requirement, previous environment.LockedPackage) bool { - if requirement.Git != nil { - return previous.Source.Git != nil && previous.Source.Git.Requested == requirement.Git.Selector - } - return previous.Source.Registry != nil && previous.Source.Registry.Requested == requirement.Registry.Selector + return requirement.RequestedSelector() == previous.Requirement().RequestedSelector() && mustSourceKey(requirement) == previous.SourceKey } func (resolver dependencyResolver) resolveGit(ctx context.Context, requirement environment.Requirement, targets []string, previous *environment.LockedPackage, update bool) (environment.LockedPackage, error) { @@ -93,61 +93,59 @@ func (resolver dependencyResolver) resolveGit(ctx context.Context, requirement e commit := source.Selector.Value resolvedRef, defaultBranch := "", "" if source.Selector.Kind != environment.SelectorCommit { - var resolution sourceacquisition.RefResolution - var err error - if named, ok := resolver.app.SourceAcquirer.(interface { - ResolveGitHubSelector(context.Context, string, string) (sourceacquisition.RefResolution, error) - ResolveGitHubRef(context.Context, string, string) (sourceacquisition.RefResolution, error) - }); ok { - selector := source.Selector.Value - if source.Selector.Kind == environment.SelectorDefault { - selector = "default" - } - if source.Selector.Kind == environment.SelectorDefault { - resolution, err = named.ResolveGitHubSelector(ctx, source.Repository, selector) - } else { - resolution, err = named.ResolveGitHubRef(ctx, source.Repository, selector) - } - } else { - return environment.LockedPackage{}, errors.New("source acquirer cannot resolve Git refs") - } + resolution, err := resolver.app.resolveDependencyGit(ctx, *source) if err != nil { return environment.LockedPackage{}, err } commit, resolvedRef, defaultBranch = resolution.Commit, resolution.ResolvedRef, resolution.DefaultBranch } - identity := githubIdentity(source.Repository, commit, source.Path) - var snapshot domain.PackageSnapshot - var err error - if previous != nil && previous.Source.Git != nil && previous.TreeDigest != "" && !update && sameRequestedSelector(requirement, *previous) { - snapshot, err = resolver.app.SourceAcquirer.AcquireGitHubVerified(ctx, source.Repository, commit, source.Path, previous.TreeDigest) - } else { - snapshot, err = resolver.app.SourceAcquirer.AcquireGitHub(ctx, source.Repository, commit, source.Path) - } + snapshot, err := resolver.app.acquireDependencyGit(ctx, *source, commit, "") if err != nil { return environment.LockedPackage{}, err } - snapshot.Source = domain.SourceIdentity{RequestedSource: identity, CanonicalSource: identity, Repository: source.Repository, - PackageSubpath: source.Path, ResolvedRevision: commit, SourceBindingHint: "uap-dependency-git"} + sourcePin := environment.LockedPackage{Source: environment.LockedSource{Git: &environment.LockedGitSource{Repository: source.Repository, URL: source.URL, Path: source.Path, Commit: commit}}} + snapshot.Source = identityFromLocked(sourcePin) loaded, err := resolver.app.loadStandardSnapshot(ctx, snapshot, domain.OriginModeDirect, nil, nil) if err != nil { return environment.LockedPackage{}, err } - if _, err := directoryv1.ResolveDirectExact(loaded.envelope.Source); err != nil { - _ = loaded.cleanup() - return environment.LockedPackage{}, err - } locked := environment.LockedPackage{ Alias: requirement.Alias, Targets: append([]string{}, targets...), SourceKey: mustSourceKey(requirement), Version: loaded.envelope.Manifest.Version, DigestAlgorithm: domain.TreeDigestAlgorithm, TreeDigest: loaded.envelope.TreeDigest, ManifestDigest: loaded.envelope.ManifestDigest, ExecutableFiles: append([]string{}, loaded.envelope.ExecutableFiles...), - Source: environment.LockedSource{Git: &environment.LockedGitSource{Repository: source.Repository, Path: source.Path, + Source: environment.LockedSource{Git: &environment.LockedGitSource{Repository: source.Repository, URL: source.URL, Path: source.Path, Requested: source.Selector, Commit: commit, ResolvedRef: resolvedRef, DefaultBranch: defaultBranch}}, } if strings.Contains(locked.Alias, "/") || locked.Alias == "" { locked.Alias = dependencyManifestAlias(loaded.envelope.Manifest.Name) } - return locked, resolver.finishLoaded(loaded) + return locked, resolver.finishLoaded(loaded, locked) +} + +func (resolver dependencyResolver) resolveArchive(ctx context.Context, requirement environment.Requirement, targets []string) (environment.LockedPackage, error) { + source := requirement.Archive + direct, err := resolver.app.directAcquirer() + if err != nil { + return environment.LockedPackage{}, err + } + snapshot, err := direct.AcquireArchive(ctx, sourcespec.Archive{URL: source.URL, Path: source.Path}, source.SHA256, "") + if err != nil { + return environment.LockedPackage{}, err + } + loaded, err := resolver.app.loadStandardSnapshot(ctx, snapshot, domain.OriginModeDirect, nil, nil) + if err != nil { + return environment.LockedPackage{}, err + } + locked := environment.LockedPackage{Alias: requirement.Alias, Targets: append([]string{}, targets...), + SourceKey: mustSourceKey(requirement), Version: loaded.envelope.Manifest.Version, DigestAlgorithm: domain.TreeDigestAlgorithm, + TreeDigest: loaded.envelope.TreeDigest, ManifestDigest: loaded.envelope.ManifestDigest, ExecutableFiles: append([]string{}, loaded.envelope.ExecutableFiles...), + Source: environment.LockedSource{Archive: &environment.LockedArchiveSource{URL: source.URL, Path: loaded.envelope.Source.PackageSubpath, + RequestedSHA256: source.SHA256, SHA256: loaded.envelope.Source.ArchiveSHA256}}} + if locked.Source.Archive.Path != source.Path { + _ = loaded.cleanup() + return environment.LockedPackage{}, errors.New("archive requirement must explicitly select its logical package path") + } + return locked, resolver.finishLoaded(loaded, locked) } func (resolver dependencyResolver) resolveRegistry(ctx context.Context, requirement environment.Requirement, targets []string) (environment.LockedPackage, error) { @@ -172,7 +170,7 @@ func (resolver dependencyResolver) resolveRegistry(ctx context.Context, requirem if strings.Contains(locked.Alias, "/") || locked.Alias == "" { locked.Alias = dependencyManifestAlias(loaded.envelope.Manifest.Name) } - return locked, resolver.finishLoaded(loaded) + return locked, resolver.finishLoaded(loaded, locked) } func registryLockedPackage(requirement environment.Requirement, loaded loadedPackage) (environment.LockedPackage, error) { @@ -193,11 +191,26 @@ func registryLockedPackage(requirement environment.Requirement, loaded loadedPac return locked, nil } -func (resolver dependencyResolver) finishLoaded(loaded loadedPackage) error { +func (resolver dependencyResolver) finishLoaded(loaded loadedPackage, pins ...environment.LockedPackage) error { if _, err := packageDependencyFacts(loaded.envelope); err != nil { _ = loaded.cleanup() return err } + if len(pins) > 0 && resolver.app.dependencySnapshots != nil { + pin := pins[0] + snapshot, err := (packagedigest.Builder{}).SnapshotWithExecutables(context.Background(), loaded.envelope.SnapshotRoot, identityFromLocked(pin), loaded.envelope.ExecutableFiles) + if err != nil { + _ = loaded.cleanup() + return err + } + key := dependencySnapshotKey{sourceKey: pin.SourceKey, revision: pin.Revision(), algorithm: pin.DigestAlgorithm, digest: pin.TreeDigest} + cache := resolver.app.dependencySnapshots + if old, ok := cache.snapshots[key]; ok { + _ = packagedigest.Remove(old) + } + cache.snapshots[key] = snapshot + return loaded.cleanup() + } if !resolver.dryRun { snapshot := domain.PackageSnapshot{Root: loaded.envelope.SnapshotRoot, TreeDigest: loaded.envelope.TreeDigest, DigestAlgorithm: domain.TreeDigestAlgorithm, ExecutableFiles: append([]string{}, loaded.envelope.ExecutableFiles...), diff --git a/cli/internal/agentpluginscli/deps_source.go b/cli/internal/agentpluginscli/deps_source.go new file mode 100644 index 000000000..45c8a030c --- /dev/null +++ b/cli/internal/agentpluginscli/deps_source.go @@ -0,0 +1,104 @@ +package agentpluginscli + +import ( + "context" + "errors" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/sourceacquisition" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +func (app App) directAcquirer() (sourceacquisition.DirectAcquirer, error) { + acquirer, ok := app.SourceAcquirer.(sourceacquisition.DirectAcquirer) + if !ok { + return nil, errors.New("direct Git and archive source acquisition is unavailable") + } + return acquirer, nil +} + +func gitSourceSpec(source environment.GitSource) sourcespec.Git { + return sourcespec.Git{URL: source.Endpoint(), Repository: source.Repository, Path: source.Path} +} + +// Historical Repository pins keep the exact old identity, even when transport +// acquisition uses native credentials. Identity is selected by the lock, never +// by the transport implementation or the first writer of a shared store tree. +func legacyGitIdentity(repository, commit, packagePath, hint string) domain.SourceIdentity { + identity := githubIdentity(repository, commit, packagePath) + return domain.SourceIdentity{RequestedSource: identity, CanonicalSource: identity, Repository: repository, + PackageSubpath: packagePath, ResolvedRevision: commit, SourceBindingHint: hint} +} + +func identityFromLocked(pin environment.LockedPackage) domain.SourceIdentity { + if source := pin.Source.Git; source != nil { + if source.Repository != "" { + return legacyGitIdentity(source.Repository, source.Commit, source.Path, "uap-dependency-git") + } + return domain.SourceIdentity{SourceKind: "git", SourceURL: source.URL, RequestedSource: source.URL, + CanonicalSource: sourcespec.Key(sourcespec.KindGit, source.URL, source.Path), PackageSubpath: source.Path, + ResolvedRevision: source.Commit, SourceBindingHint: "direct-git"} + } + if source := pin.Source.Archive; source != nil { + return domain.SourceIdentity{SourceKind: "archive", SourceURL: source.URL, ArchiveSHA256: source.SHA256, + RequestedSource: source.URL, CanonicalSource: sourcespec.Key(sourcespec.KindArchive, source.URL, source.Path), + PackageSubpath: source.Path, ResolvedRevision: "sha256:" + source.SHA256, SourceBindingHint: "direct-archive"} + } + if source := pin.Source.Registry; source != nil { + return legacyGitIdentity(source.Repository, source.Commit, source.Path, "uap-dependency-registry") + } + return domain.SourceIdentity{} +} + +func (app App) acquireDependencyGit(ctx context.Context, source environment.GitSource, commit, digest string) (domain.PackageSnapshot, error) { + if direct, ok := app.SourceAcquirer.(sourceacquisition.DirectAcquirer); ok { + return direct.AcquireGit(ctx, gitSourceSpec(source), commit, digest) + } + if source.URL != "" { + return domain.PackageSnapshot{}, errors.New("git URL acquisition is unavailable") + } + if digest != "" { + return app.SourceAcquirer.AcquireGitHubVerified(ctx, source.Repository, commit, source.Path, digest) + } + return app.SourceAcquirer.AcquireGitHub(ctx, source.Repository, commit, source.Path) +} + +func (app App) acquirePinnedSource(ctx context.Context, pin environment.LockedPackage) (domain.PackageSnapshot, error) { + if source := pin.Source.Git; source != nil { + return app.acquireDependencyGit(ctx, environment.GitSource{Repository: source.Repository, URL: source.URL, Path: source.Path}, source.Commit, pin.TreeDigest) + } + if source := pin.Source.Archive; source != nil { + direct, err := app.directAcquirer() + if err != nil { + return domain.PackageSnapshot{}, err + } + return direct.AcquireArchive(ctx, sourcespec.Archive{URL: source.URL, Path: source.Path}, source.SHA256, pin.TreeDigest) + } + if source := pin.Source.Registry; source != nil { + // Signed Directory remains on its anonymous, immutable GitHub contract. + return app.SourceAcquirer.AcquireGitHubVerified(ctx, source.Repository, source.Commit, source.Path, pin.TreeDigest) + } + return domain.PackageSnapshot{}, errors.New("locked package has no source") +} + +func (app App) resolveDependencyGit(ctx context.Context, source environment.GitSource) (sourceacquisition.RefResolution, error) { + if direct, ok := app.SourceAcquirer.(sourceacquisition.DirectAcquirer); ok { + selector := source.Selector.Value + if source.Selector.Kind == environment.SelectorDefault { + selector = "default" + } + return direct.ResolveGit(ctx, gitSourceSpec(source), selector, source.Selector.Kind != environment.SelectorDefault) + } + named, ok := app.SourceAcquirer.(interface { + ResolveGitHubSelector(context.Context, string, string) (sourceacquisition.RefResolution, error) + ResolveGitHubRef(context.Context, string, string) (sourceacquisition.RefResolution, error) + }) + if !ok || source.URL != "" { + return sourceacquisition.RefResolution{}, errors.New("source acquirer cannot resolve Git refs") + } + if source.Selector.Kind == environment.SelectorDefault { + return named.ResolveGitHubSelector(ctx, source.Repository, "default") + } + return named.ResolveGitHubRef(ctx, source.Repository, source.Selector.Value) +} diff --git a/cli/internal/agentpluginscli/deps_staging.go b/cli/internal/agentpluginscli/deps_staging.go index 9a98541d8..da12e05d8 100644 --- a/cli/internal/agentpluginscli/deps_staging.go +++ b/cli/internal/agentpluginscli/deps_staging.go @@ -10,7 +10,7 @@ import ( ) type dependencySnapshotKey struct { - repository, commit, path, algorithm, digest string + sourceKey, revision, algorithm, digest string } func lockedDependencyCoordinates(locked environment.LockedPackage) (string, string, string, error) { @@ -38,11 +38,11 @@ func (cache *dependencySnapshotCache) close() { } func (app App) acquireLockedSnapshot(ctx context.Context, locked environment.LockedPackage) (domain.PackageSnapshot, error) { - repository, commit, path, err := lockedDependencyCoordinates(locked) - if err != nil { - return domain.PackageSnapshot{}, err + sourceKey := locked.SourceKey + if sourceKey == "" { + sourceKey = identityFromLocked(locked).CanonicalSource } - key := dependencySnapshotKey{repository: repository, commit: commit, path: path, algorithm: locked.DigestAlgorithm, digest: locked.TreeDigest} + key := dependencySnapshotKey{sourceKey: sourceKey, revision: locked.Revision(), algorithm: locked.DigestAlgorithm, digest: locked.TreeDigest} cache := app.dependencySnapshots var acquired domain.PackageSnapshot var ok bool @@ -51,10 +51,11 @@ func (app App) acquireLockedSnapshot(ctx context.Context, locked environment.Loc } if !ok { var err error - acquired, err = app.SourceAcquirer.AcquireGitHubVerified(ctx, key.repository, key.commit, key.path, key.digest) + acquired, err = app.acquirePinnedSource(ctx, locked) if err != nil { return domain.PackageSnapshot{}, err } + acquired.Source = identityFromLocked(locked) if cache == nil { return acquired, nil } @@ -66,7 +67,7 @@ func (app App) acquireLockedSnapshot(ctx context.Context, locked environment.Loc } // A consumer may release its snapshot, while the command retains the raw // acquisition. Recompute the digest rather than trusting retained metadata. - snapshot, err := (packagedigest.Builder{}).SnapshotWithExecutables(ctx, acquired.Root, acquired.Source, acquired.ExecutableFiles) + snapshot, err := (packagedigest.Builder{}).SnapshotWithExecutables(ctx, acquired.Root, identityFromLocked(locked), acquired.ExecutableFiles) if err != nil { return domain.PackageSnapshot{}, err } diff --git a/cli/internal/agentpluginscli/deps_status.go b/cli/internal/agentpluginscli/deps_status.go index da24bbee0..e9c1b36fb 100644 --- a/cli/internal/agentpluginscli/deps_status.go +++ b/cli/internal/agentpluginscli/deps_status.go @@ -171,13 +171,7 @@ func findInspectedEnvironmentBinding(view installer.Inspection, receipt domain.E } func lockedCommit(pkg environment.LockedPackage) string { - if pkg.Source.Git != nil { - return pkg.Source.Git.Commit - } - if pkg.Source.Registry != nil { - return pkg.Source.Registry.Commit - } - return "" + return pkg.Revision() } func joinUnique(values []string) string { diff --git a/cli/internal/agentpluginscli/deps_sync.go b/cli/internal/agentpluginscli/deps_sync.go index 057db4105..b2abe39b0 100644 --- a/cli/internal/agentpluginscli/deps_sync.go +++ b/cli/internal/agentpluginscli/deps_sync.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "github.com/spf13/cobra" @@ -12,6 +13,7 @@ import ( "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/installer" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" ) func materializeLockedPackage(ctx context.Context, app App, locked environment.LockedPackage, publish bool, recorded *domain.RecordedDirectoryRelease, operation ...domain.DirectoryOperation) (loadedPackage, error) { @@ -21,17 +23,11 @@ func materializeLockedPackage(ctx context.Context, app App, locked environment.L } else if !errors.Is(err, environmentstore.ErrMissing) { return loadedPackage{}, err } - repository, commit, packagePath, err := lockedDependencyCoordinates(locked) - if err != nil { - return loadedPackage{}, err - } snapshot, err := app.acquireLockedSnapshot(ctx, locked) if err != nil { return loadedPackage{}, err } - identity := githubIdentity(repository, commit, packagePath) - snapshot.Source = domain.SourceIdentity{RequestedSource: identity, CanonicalSource: identity, Repository: repository, - PackageSubpath: packagePath, ResolvedRevision: commit, SourceBindingHint: dependencySourceHint(locked)} + snapshot.Source = identityFromLocked(locked) origin, directory := domain.OriginModeDirect, (*domain.DirectoryOrigin)(nil) if locked.Source.Registry != nil { origin = domain.OriginModeDirectory @@ -82,14 +78,7 @@ func dependencySourceHint(locked environment.LockedPackage) string { } func loadStoredDependency(ctx context.Context, app App, pkg environmentstore.Package, locked environment.LockedPackage, recorded *domain.RecordedDirectoryRelease, operation ...domain.DirectoryOperation) (loadedPackage, error) { - var repository, commit, subpath string - if locked.Source.Git != nil { - repository, commit, subpath = locked.Source.Git.Repository, locked.Source.Git.Commit, locked.Source.Git.Path - } else { - repository, commit, subpath = locked.Source.Registry.Repository, locked.Source.Registry.Commit, locked.Source.Registry.Path - } - identity := githubIdentity(repository, commit, subpath) - pkg.Metadata.Source = domain.SourceIdentity{RequestedSource: identity, CanonicalSource: identity, Repository: repository, PackageSubpath: subpath, ResolvedRevision: commit, SourceBindingHint: dependencySourceHint(locked)} + pkg.Metadata.Source = identityFromLocked(locked) input := domain.LoadInput{SnapshotRoot: pkg.Root, TreeDigest: pkg.Metadata.TreeDigest, ExecutableFiles: append([]string{}, pkg.Metadata.ExecutableFiles...), Source: pkg.Metadata.Source} envelope, err := app.loadStandardEnvelope(ctx, input) @@ -462,7 +451,15 @@ func buildDependencyRepairs(ctx context.Context, app App, state domain.StateFile return nil, errors.New("recorded package revision is missing") } pkg := environment.LockedPackage{Alias: authority.Alias, SourceKey: authority.SourceKey, DigestAlgorithm: domain.TreeDigestAlgorithm, TreeDigest: rev.TreeDigest, ManifestDigest: rev.ManifestDigest, Targets: []string{string(target)}} - pkg.Source.Git = &environment.LockedGitSource{Repository: installation.Source.Repository, Path: installation.Source.PackageSubpath, Commit: rev.ResolvedRevision, Requested: environment.Selector{Kind: environment.SelectorCommit, Value: rev.ResolvedRevision}} + pkg.Source.Git = &environment.LockedGitSource{Repository: installation.Source.Repository, Path: installation.Source.PackageSubpath, Commit: rev.ResolvedRevision, URL: installation.Source.SourceURL, Requested: environment.Selector{Kind: environment.SelectorCommit, Value: rev.ResolvedRevision}} + if installation.Source.SourceKind == "archive" { + digest := strings.TrimPrefix(rev.ResolvedRevision, "sha256:") + if !sourcespec.ValidSHA256(digest) { + return nil, errors.New("recorded archive revision is invalid") + } + pkg.Source.Git = nil + pkg.Source.Archive = &environment.LockedArchiveSource{URL: installation.Source.SourceURL, Path: installation.Source.PackageSubpath, SHA256: digest, RequestedSHA256: digest} + } if installation.Directory != nil { pkg.Source.Git = nil pkg.Source.Registry = &environment.LockedRegistrySource{Distribution: installation.Directory.DistributionID, DistributionID: installation.Directory.DistributionID, ProductID: installation.Directory.ProductID, ReleaseSequence: rev.ReleaseSequence, Repository: installation.Source.Repository, Path: installation.Source.PackageSubpath, Commit: rev.ResolvedRevision} diff --git a/cli/internal/agentpluginscli/lifecycle.go b/cli/internal/agentpluginscli/lifecycle.go index 56cedacbb..533a23c42 100644 --- a/cli/internal/agentpluginscli/lifecycle.go +++ b/cli/internal/agentpluginscli/lifecycle.go @@ -693,6 +693,9 @@ func environmentAuthorityForBinding(state domain.StateFileV2, installationID str } func updateSource(installation domain.Installation) string { + if installation.Source.SourceKind != "" && installation.Source.RequestedSource == "" { + return directSourceRequested(installation.Source) + } if installation.Source.Repository == "" && filepath.IsAbs(installation.Source.CanonicalSource) { return installation.Source.CanonicalSource } diff --git a/cli/internal/agentpluginscli/saved_add.go b/cli/internal/agentpluginscli/saved_add.go index 99aa51853..d11be1215 100644 --- a/cli/internal/agentpluginscli/saved_add.go +++ b/cli/internal/agentpluginscli/saved_add.go @@ -15,6 +15,7 @@ import ( "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" ) func routeSavedAdd(cmd *cobra.Command, app App, opts *options, source string, activation, auth bool) (bool, error) { @@ -22,7 +23,7 @@ func routeSavedAdd(cmd *cobra.Command, app App, opts *options, source string, ac switch { case opts.noSave: reason = "no_save_requested" - case explicitLocalPath(source): + case localArchiveInput(source) || explicitLocalPath(source): reason = "local_source_not_portable" case strings.HasPrefix(source, "discovery:"): reason = "discovery_source_not_saved" @@ -46,7 +47,9 @@ func routeSavedAdd(cmd *cobra.Command, app App, opts *options, source string, ac } matches := exactGitPattern.FindStringSubmatch(source) if matches == nil && !isDirectorySelector(source) { - return true, errors.New("saved add requires an exact GitHub commit or Directory selector; use --no-save") + if _, err := sourcespec.Parse(source); err != nil { + return true, err + } } targets, err = savedAddTargets(cmd, app, opts, source, targets) if err != nil { @@ -70,6 +73,9 @@ func cloneSavedRequirements(value environment.Requirements) environment.Requirem } func resolveSavedAdd(cmd *cobra.Command, app App, opts *options, source string, targets []domain.ClientID, matches []string, snapshot environmentSnapshot) (environment.Requirement, environment.LockedPackage, error) { + if matches == nil && !isDirectorySelector(source) { + return resolveDirectSavedAdd(cmd, app, opts, source, snapshot) + } var requirement environment.Requirement alias := "" var pin environment.LockedPackage @@ -161,7 +167,7 @@ func acquireSavedAdd(cmd *cobra.Command, app App, opts *options, source string, pin.ManifestDigest = loaded.envelope.ManifestDigest pin.Version = loaded.envelope.Manifest.Version pin.ExecutableFiles = append([]string{}, loaded.envelope.ExecutableFiles...) - if err := (dependencyResolver{app: app, dryRun: opts.dryRun}).finishLoaded(loaded); err != nil { + if err := (dependencyResolver{app: app, dryRun: opts.dryRun}).finishLoaded(loaded, pin); err != nil { return environment.Requirement{}, environment.LockedPackage{}, err } return requirement, pin, nil @@ -176,7 +182,7 @@ func rejectKnownOwnedAdd(app App, source string) error { if err != nil { return err } - key := "" + key := savedDirectSourceKey(source) if matches := exactGitPattern.FindStringSubmatch(source); matches != nil { key = "github:" + strings.ToLower(matches[1]) + ":" + matches[3] } @@ -242,7 +248,7 @@ func seedSavedAdd(snapshot environmentSnapshot, requirement environment.Requirem if pin.TreeDigest == "" { return previous } - if !reflect.DeepEqual(snapshot.Requirements.Dependencies[alias], requirement) && pin.Source.Git != nil && requirement.Git != nil && pin.Source.Git.Commit != requirement.Git.Selector.Value { + if !reflect.DeepEqual(snapshot.Requirements.Dependencies[alias], requirement) && pin.Source.Git != nil && requirement.Git != nil && requirement.Git.Selector.Kind == environment.SelectorCommit && pin.Source.Git.Commit != requirement.Git.Selector.Value { return previous } for i, pkg := range previous.Packages { @@ -260,7 +266,7 @@ func loadSavedAddPackage(cmd *cobra.Command, app App, source string, targets []d var err error var loaded loadedPackage if matches != nil { - snapshot, e := app.SourceAcquirer.AcquireGitHub(cmd.Context(), requirement.Git.Repository, requirement.Git.Selector.Value, requirement.Git.Path) + snapshot, e := app.acquireDependencyGit(cmd.Context(), *requirement.Git, requirement.Git.Selector.Value, "") if e != nil { return loadedPackage{}, e } diff --git a/cli/internal/agentpluginscli/saved_add_e2e_test.go b/cli/internal/agentpluginscli/saved_add_e2e_test.go index 8e2303e77..ba2bf1eaa 100644 --- a/cli/internal/agentpluginscli/saved_add_e2e_test.go +++ b/cli/internal/agentpluginscli/saved_add_e2e_test.go @@ -19,6 +19,7 @@ import ( "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/installer" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" legacyports "github.com/777genius/plugin-kit-ai/install/integrationctl/ports" ) @@ -44,6 +45,19 @@ func (s *savedJourneySource) ResolveGitHubSelector(ctx context.Context, repo, se return s.Acquirer.ResolveGitHubSelector(ctx, repo, selector) } +func (s *savedJourneySource) AcquireGit(ctx context.Context, source sourcespec.Git, commit, digest string) (domain.PackageSnapshot, error) { + s.downloads++ + return s.Acquirer.AcquireGit(ctx, source, commit, digest) +} +func (s *savedJourneySource) ResolveGit(ctx context.Context, source sourcespec.Git, selector string, exact bool) (sourceacquisition.RefResolution, error) { + if exact { + s.refs++ + } else { + s.selectors++ + } + return s.Acquirer.ResolveGit(ctx, source, selector, exact) +} + type savedJourneyRunner struct { inner dependencyClaudeListing calls int diff --git a/cli/internal/agentpluginscli/saved_direct.go b/cli/internal/agentpluginscli/saved_direct.go new file mode 100644 index 000000000..a9504d825 --- /dev/null +++ b/cli/internal/agentpluginscli/saved_direct.go @@ -0,0 +1,142 @@ +package agentpluginscli + +import ( + "errors" + "net/url" + + "github.com/spf13/cobra" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +func directRequirement(spec sourcespec.Spec) environment.Requirement { + requirement := environment.Requirement{Alias: "candidate"} + if source := spec.Git; source != nil && source.Browser == nil { + selector := environment.Selector{Kind: environment.SelectorDefault} + if source.Ref != "" { + selector = environment.Selector{Kind: environment.SelectorRef, Value: source.Ref} + } + if source.Commit != "" { + selector = environment.Selector{Kind: environment.SelectorCommit, Value: source.Commit} + } + requirement.Git = &environment.GitSource{Repository: source.Repository, Path: source.Path, Selector: selector} + if source.Repository == "" { + requirement.Git.URL = source.URL + } + } + if source := spec.Archive; source != nil && spec.Kind == sourcespec.KindArchive { + requirement.Archive = &environment.ArchiveSource{URL: source.URL, Path: source.Path, SHA256: source.SHA256} + } + return requirement +} + +func retainedDirectAdd(snapshot environmentSnapshot, requirement environment.Requirement, requestedAlias string) (environment.Requirement, environment.LockedPackage, bool, error) { + if requirement.Git == nil && requirement.Archive == nil { + return requirement, environment.LockedPackage{}, false, nil + } + key := mustSourceKey(requirement) + for _, pin := range snapshot.Lock.Packages { + if pin.SourceKey != key { + continue + } + alias := dependencyRootKey(snapshot.Requirements, snapshot.Lock, pin.Alias) + if alias == "" { + return requirement, pin, false, nil + } + if requestedAlias != "" && requestedAlias != alias { + return requirement, pin, true, errors.New("source_change: source already saved under alias " + alias) + } + old := snapshot.Requirements.Dependencies[alias] + same := old.RequestedSelector() == requirement.RequestedSelector() + if requirement.Git != nil && pin.Source.Git != nil && requirement.Git.Selector.Kind == environment.SelectorCommit { + same = same || requirement.Git.Selector.Value == pin.Source.Git.Commit + } + if !same { + return requirement, pin, true, errors.New("saved selector must be changed with update " + alias + " or an explicit uap.toml edit") + } + return old, pin, true, nil + } + return requirement, environment.LockedPackage{}, false, nil +} + +func resolveDirectSavedAdd(cmd *cobra.Command, app App, opts *options, source string, snapshot environmentSnapshot) (environment.Requirement, environment.LockedPackage, error) { + spec, err := sourcespec.Parse(source) + if err != nil { + return environment.Requirement{}, environment.LockedPackage{}, err + } + requirement := directRequirement(spec) + retained, pin, found, err := retainedDirectAdd(snapshot, requirement, opts.alias) + if found || err != nil { + return retained, pin, err + } + result, err := app.acquireDirectPackage(cmd.Context(), spec) + if err != nil { + return environment.Requirement{}, environment.LockedPackage{}, err + } + defer func() { _ = result.loaded.cleanup() }() + requirement = result.requirement + requirement.Alias = "candidate" + retained, pin, found, err = retainedDirectAdd(snapshot, requirement, opts.alias) + if found || err != nil { + return retained, pin, err + } + alias := opts.alias + if alias == "" { + alias = dependencyManifestAlias(result.loaded.envelope.Manifest.Name) + } + requirement.Alias = alias + if err := requirement.Validate(); err != nil { + return requirement, environment.LockedPackage{}, err + } + if old, exists := snapshot.Requirements.Dependencies[alias]; exists && mustSourceKey(old) != mustSourceKey(requirement) { + return requirement, environment.LockedPackage{}, errors.New("alias already belongs to another source; use --alias") + } + result.requirement = requirement + pin = completeDirectPin(result, alias) + if err := (dependencyResolver{app: app, dryRun: opts.dryRun}).finishLoaded(result.loaded, pin); err != nil { + return requirement, pin, err + } + return requirement, pin, nil +} + +func completeDirectPin(result directPackage, alias string) environment.LockedPackage { + pin := result.pin + pin.Alias, pin.SourceKey = alias, mustSourceKey(result.requirement) + pin.Version = result.loaded.envelope.Manifest.Version + pin.DigestAlgorithm, pin.TreeDigest = domain.TreeDigestAlgorithm, result.loaded.envelope.TreeDigest + pin.ManifestDigest = result.loaded.envelope.ManifestDigest + pin.ExecutableFiles = append([]string{}, result.loaded.envelope.ExecutableFiles...) + return pin +} + +func savedDirectSourceKey(raw string) string { + spec, err := sourcespec.Parse(raw) + if err != nil { + return "" + } + requirement := directRequirement(spec) + if requirement.Git == nil && requirement.Archive == nil { + return "" + } + return mustSourceKey(requirement) +} + +func directSourceRequested(source domain.SourceBinding) string { + if source.SourceKind == "archive" { + return source.SourceURL + "#sha256=" + source.ArchiveSHA256 + archivePathSelector(source.PackageSubpath) + } + value := source.SourceURL + "#" + source.ResolvedRevision + if source.PackageSubpath != "" { + value += "//" + source.PackageSubpath + } + return value +} +func archivePathSelector(packagePath string) string { + if packagePath == "" { + return "" + } + // Archive paths are already portable; the parser decodes fragment values. + return "&path=" + url.QueryEscape(packagePath) +} diff --git a/cli/internal/agentpluginscli/saved_lifecycle.go b/cli/internal/agentpluginscli/saved_lifecycle.go index 347fbd25a..161359ae8 100644 --- a/cli/internal/agentpluginscli/saved_lifecycle.go +++ b/cli/internal/agentpluginscli/saved_lifecycle.go @@ -294,7 +294,7 @@ func (selection savedLifecycleSelection) update(cmd *cobra.Command, app App, opt return updateSavedRegistry(cmd, app, opts, snapshot, selection.alias, selection.installation) } updateAlias := key - if requirement.Git != nil && requirement.Git.Selector.Kind == environment.SelectorCommit { + if (requirement.Git != nil && requirement.Git.Selector.Kind == environment.SelectorCommit) || (requirement.Archive != nil && requirement.Archive.SHA256 != "") { updateAlias = "" } locked, err := (environment.Coordinator{Resolver: dependencyResolver{requirements: requirements, app: app, dryRun: opts.dryRun}}).Resolve(cmd.Context(), requirements, snapshot.Lock, updateAlias) @@ -319,6 +319,10 @@ func (selection savedLifecycleSelection) renderNextAction(cmd *cobra.Command, op _, err := fmt.Fprintln(cmd.OutOrStdout(), "Exact commit pin retained; choose a new pin with agentplugins add @//path") return err } + if operation == "update" && requirement.Archive != nil && requirement.Archive.SHA256 != "" { + _, err := fmt.Fprintln(cmd.OutOrStdout(), "Exact archive checksum retained; choose a new checksum in uap.toml") + return err + } if operation == "remove" && selection.installation != nil && len(savedAliasTargets(requirements, domain.StateFileV2{}, selection.alias)) == 0 { _, err := fmt.Fprintf(cmd.OutOrStdout(), "PLUGIN_DATA preserved; after removal completes, run agentplugins remove %s --purge-data to delete retained data\n", selection.installation.InstallationID) return err @@ -350,8 +354,9 @@ func updateSavedRegistry(cmd *cobra.Command, app App, opts *options, snapshot en } return requirements, snapshot.Lock, err } - requirements, _ = savedRegistryCandidate(snapshot, alias, old, loaded) - if err := (dependencyResolver{app: app, dryRun: opts.dryRun}).finishLoaded(loaded); err != nil { + requirements, candidate := savedRegistryCandidate(snapshot, alias, old, loaded) + next, _ := candidate.Package(alias) + if err := (dependencyResolver{app: app, dryRun: opts.dryRun}).finishLoaded(loaded, next); err != nil { return requirements, snapshot.Lock, err } // Validate the authoritative old graph before rebuilding edges from the new diff --git a/cli/internal/agentpluginscli/source.go b/cli/internal/agentpluginscli/source.go index bf70e9fee..adea48893 100644 --- a/cli/internal/agentpluginscli/source.go +++ b/cli/internal/agentpluginscli/source.go @@ -16,6 +16,7 @@ import ( "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" clientplanner "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/planner" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" ) var ( @@ -252,6 +253,13 @@ func (app App) loadPackageFor(ctx context.Context, raw string, request packageRe if requested == "" { return loadedPackage{}, fmt.Errorf("plugin name or source is required") } + if localArchiveInput(requested) { + spec, err := sourcespec.Parse(requested) + if err != nil { + return loadedPackage{}, err + } + return app.acquireDirectInput(ctx, spec) + } if explicitLocalPath(requested) { return app.acquireLocal(ctx, requested) } @@ -262,7 +270,11 @@ func (app App) loadPackageFor(ctx context.Context, raw string, request packageRe return app.acquireDiscovery(ctx, requested, request) } if !isDirectorySelector(requested) { - return loadedPackage{}, fmt.Errorf("source must be a local path, an exact owner/repo@FULL_SHA[//path] source, a discovery: slug, or a Directory selector") + spec, err := sourcespec.Parse(requested) + if err != nil { + return loadedPackage{}, err + } + return app.acquireDirectInput(ctx, spec) } return app.acquireDirectory(ctx, requested, request) } diff --git a/cli/internal/agentpluginscli/source_direct.go b/cli/internal/agentpluginscli/source_direct.go new file mode 100644 index 000000000..b38e2a0a3 --- /dev/null +++ b/cli/internal/agentpluginscli/source_direct.go @@ -0,0 +1,135 @@ +package agentpluginscli + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/sourceacquisition" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +type directPackage struct { + loaded loadedPackage + requirement environment.Requirement + pin environment.LockedPackage +} + +func localArchiveInput(raw string) bool { + spec, err := sourcespec.Parse(raw) + if err != nil || spec.Kind != sourcespec.KindLocalArchive { + return false + } + info, err := os.Stat(raw) + return err != nil || !info.IsDir() +} + +func (app App) acquireDirectPackage(ctx context.Context, spec sourcespec.Spec) (directPackage, error) { + direct, err := app.directAcquirer() + if err != nil { + return directPackage{}, err + } + if spec.Archive != nil { + snapshot, archiveErr := direct.AcquireArchive(ctx, *spec.Archive, spec.Archive.SHA256, "") + if archiveErr == nil { + return app.loadDirectArchive(ctx, snapshot, *spec.Archive) + } + if spec.Kind != sourcespec.KindHTTPSAuto || !errors.Is(archiveErr, sourceacquisition.ErrNotArchive) { + return directPackage{}, archiveErr + } + endpoint, repository, normalizeErr := sourcespec.NormalizeGitURL(spec.Archive.URL) + if normalizeErr != nil { + return directPackage{}, normalizeErr + } + spec.Git = &sourcespec.Git{URL: endpoint, Repository: repository} + } + if spec.Git == nil { + return directPackage{}, errors.New("direct source has no acquisition coordinates") + } + return app.loadDirectGit(ctx, *spec.Git) +} + +func (app App) loadDirectArchive(ctx context.Context, snapshot domain.PackageSnapshot, requested sourcespec.Archive) (directPackage, error) { + loaded, err := app.loadStandardSnapshot(ctx, snapshot, domain.OriginModeDirect, nil, nil) + if err != nil { + return directPackage{}, err + } + source := loaded.envelope.Source + requirement := environment.Requirement{Archive: &environment.ArchiveSource{URL: source.SourceURL, Path: source.PackageSubpath, SHA256: requested.SHA256}} + pin := environment.LockedPackage{Source: environment.LockedSource{Archive: &environment.LockedArchiveSource{URL: source.SourceURL, Path: source.PackageSubpath, RequestedSHA256: requested.SHA256, SHA256: source.ArchiveSHA256}}} + return directPackage{loaded: loaded, requirement: requirement, pin: pin}, nil +} + +func (app App) loadDirectGit(ctx context.Context, source sourcespec.Git) (directPackage, error) { + direct, err := app.directAcquirer() + if err != nil { + return directPackage{}, err + } + resolved, err := direct.ResolveGitSource(ctx, source) + if err != nil { + return directPackage{}, err + } + git := resolved.Git + snapshot, err := app.acquireDirectGitRoot(ctx, git) + if err != nil { + return directPackage{}, err + } + git.Path = snapshot.Source.PackageSubpath + selector := environment.Selector{Kind: environment.SelectorDefault} + if git.Ref != "" { + selector = environment.Selector{Kind: environment.SelectorRef, Value: git.Ref} + } + if source.Commit != "" || (source.Browser != nil && git.Ref == "") { + selector = environment.Selector{Kind: environment.SelectorCommit, Value: git.Commit} + } + requirement := environment.Requirement{Git: &environment.GitSource{Repository: git.Repository, Path: git.Path, Selector: selector}} + if git.Repository == "" { + requirement.Git.URL = git.URL + } + pin := environment.LockedPackage{Source: environment.LockedSource{Git: &environment.LockedGitSource{Repository: requirement.Git.Repository, URL: requirement.Git.URL, Path: git.Path, Commit: git.Commit, Requested: selector, ResolvedRef: resolved.Resolution.ResolvedRef, DefaultBranch: resolved.Resolution.DefaultBranch}}} + snapshot.Source = identityFromLocked(pin) + loaded, err := app.loadStandardSnapshot(ctx, snapshot, domain.OriginModeDirect, nil, nil) + if err != nil { + return directPackage{}, err + } + return directPackage{loaded: loaded, requirement: requirement, pin: pin}, nil +} + +func (app App) acquireDirectGitRoot(ctx context.Context, git sourcespec.Git) (domain.PackageSnapshot, error) { + direct, err := app.directAcquirer() + if err != nil { + return domain.PackageSnapshot{}, err + } + snapshot, err := direct.AcquireGit(ctx, git, git.Commit, "") + if err != nil { + return domain.PackageSnapshot{}, err + } + if _, rootErr := os.Stat(filepath.Join(snapshot.Root, "plugin.json")); !os.IsNotExist(rootErr) || git.Path != "" { + return snapshot, nil + } + _ = packagedigest.Remove(snapshot) + paths, err := direct.DiscoverGitPackages(ctx, git, git.Commit) + if err != nil { + return domain.PackageSnapshot{}, err + } + if len(paths) != 1 { + return domain.PackageSnapshot{}, fmt.Errorf("choose an explicit package path; standard package candidates: %v", paths) + } + git.Path = paths[0] + return direct.AcquireGit(ctx, git, git.Commit, "") +} + +func (app App) acquireDirectInput(ctx context.Context, spec sourcespec.Spec) (loadedPackage, error) { + result, err := app.acquireDirectPackage(ctx, spec) + if err != nil { + return loadedPackage{}, err + } + result.loaded.envelope.Source.RequestedSource = spec.Requested + app.warnDirectRevocation(result.loaded.envelope.TreeDigest) + return result.loaded, nil +} diff --git a/cli/internal/agentpluginscli/source_identity_e2e_test.go b/cli/internal/agentpluginscli/source_identity_e2e_test.go new file mode 100644 index 000000000..8b36bcd96 --- /dev/null +++ b/cli/internal/agentpluginscli/source_identity_e2e_test.go @@ -0,0 +1,145 @@ +package agentpluginscli + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" +) + +func TestDirectSourcesWithIdenticalBytesKeepSelectedLockProvenance(t *testing.T) { + for _, order := range [][]string{{"a", "b"}, {"b", "a"}} { + t.Run(strings.Join(order, "-"), func(t *testing.T) { + transport := newDirectTransportFixture(t, "zip", false) + sharedStore := filepath.Join(t.TempDir(), "shared-store") + coldStore := filepath.Join(t.TempDir(), "shared-cold-store") + var previousDigest, previousKey string + for _, name := range order { + fixture := directCLIEnvironment(t, transport) + fixture.app.EnvironmentStoreRoot = sharedStore + endpoint := transport.server.URL + "/" + name + ".zip" + dependencyRun(t, fixture, "add", endpoint, "--alias", name, "--target", "gemini", "--format", "json") + pin := directLockedPin(t, fixture) + if previousDigest != "" && (previousDigest != pin.TreeDigest || previousKey == pin.SourceKey) { + t.Fatal("same-byte source facts collapsed") + } + previousDigest, previousKey = pin.TreeDigest, pin.SourceKey + loaded, err := materializeLockedPackage(context.Background(), fixture.app, pin, false, nil) + if err != nil { + t.Fatal(err) + } + if loaded.envelope.Source != identityFromLocked(pin) { + t.Fatalf("store supplied first-writer identity: %+v", loaded.envelope.Source) + } + replay, err := describePinnedDependency(context.Background(), fixture.app, pin) + if err != nil || replay.Package.SourceKey != pin.SourceKey { + t.Fatalf("replay provenance changed: %+v %v", replay, err) + } + state, err := fixture.store.Load() + if err != nil { + t.Fatal(err) + } + if state.Installations[0].Source.SourceURL != endpoint { + t.Fatal("installation adopted another endpoint's provenance") + } + directColdRestoreToStore(t, fixture, transport, pin, coldStore) + } + }) + } +} + +func directColdRestoreToStore(t *testing.T, original cliFixture, transport *directTransportFixture, pin environment.LockedPackage, store string) { + t.Helper() + fresh := directCLIEnvironment(t, transport) + fresh.app.EnvironmentStoreRoot = store + oldManifest, oldLock := original.app.dependencyPaths() + newManifest, newLock := fresh.app.dependencyPaths() + dependencyWrite(t, newManifest, dependencyRead(t, oldManifest)) + dependencyWrite(t, newLock, dependencyRead(t, oldLock)) + dependencyRun(t, fresh, "deps", "sync", "--locked", "--yes", "--format", "json") + loaded, err := materializeLockedPackage(context.Background(), fresh.app, pin, false, nil) + if err != nil { + t.Fatal(err) + } + if loaded.envelope.Source != identityFromLocked(pin) { + t.Fatal("cold restore reused another endpoint's provenance") + } +} + +func TestDirectSavedPreviewDoesNotPublishStoreOrIntent(t *testing.T) { + for _, mode := range []string{"dry-run", "decline"} { + t.Run(mode, func(t *testing.T) { + transport := newDirectTransportFixture(t, "zip", false) + fixture := directCLIEnvironment(t, transport) + dependencyRun(t, fixture, "deps", "init", "--format", "json") + manifestPath, lockPath := fixture.app.dependencyPaths() + before := dependencyRead(t, manifestPath) + args := []string{"add", transport.server.URL + "/package.zip"} + if mode == "dry-run" { + args = append(args, "--target", "gemini", "--dry-run", "--format", "json") + } + _, _, err := fixture.executeInput(mode == "decline", "n\n", args...) + if err != nil { + t.Fatal(err) + } + if dependencyRead(t, manifestPath) != before { + t.Fatal("unconfirmed preview changed intent") + } + for _, path := range []string{lockPath, fixture.app.EnvironmentStoreRoot} { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("unconfirmed preview published %s: %v", filepath.Base(path), err) + } + } + state, err := fixture.store.Load() + if err != nil { + t.Fatal(err) + } + if len(state.Installations) != 0 || len(state.EnvironmentReceipts) != 0 { + t.Fatal("unconfirmed preview changed ownership") + } + }) + } +} + +func TestLegacyGitHubLockIdentitySurvivesNativeTransport(t *testing.T) { + repo := dependencyRepo(t) + fixture, source, _ := dependencySpecHarness(t, repo, dependencyClientSpec{id: domain.ClientGemini, activation: "active"}) + commit := savedJourneyGit(t, repo, "rev-parse", "HEAD") + raw := githubIdentity("acme/demo", commit, "") + dependencyRun(t, fixture, "add", raw, "--target", "gemini", "--format", "json") + pin := directLockedPin(t, fixture) + state, err := fixture.store.Load() + if err != nil { + t.Fatal(err) + } + before := state.Installations[0] + expected := legacyGitIdentity("acme/demo", commit, "", "uap-dependency-git") + if before.Source.SourceBindingID != domain.ComputeSourceBindingID(expected) || before.Source.SourceKind != "" || before.Source.SourceURL != "" { + t.Fatalf("legacy identity changed: %+v", before.Source) + } + native, err := source.AcquireGit(context.Background(), gitSourceSpec(*pin.Requirement().Git), commit, pin.TreeDigest) + if err != nil { + t.Fatal(err) + } + defer func() { _ = packagedigest.Remove(native) }() + // Store metadata can originate from the generic transport. The lock still + // supplies the historical identity on every replay and acquired-cache path. + fixture.app.EnvironmentStoreRoot = filepath.Join(t.TempDir(), "different-first-writer-store") + if _, err := fixture.app.dependencyStore().Put(context.Background(), native); err != nil { + t.Fatal(err) + } + dependencyRun(t, fixture, "repair", before.InstallationID, "--target", "gemini", "--format", "json") + dependencyRun(t, fixture, "deps", "sync", "--locked", "--yes", "--format", "json") + after, err := fixture.store.Load() + if err != nil { + t.Fatal(err) + } + if after.Installations[0].InstallationID != before.InstallationID || after.Installations[0].Source != before.Source { + t.Fatalf("legacy binding changed: %+v", after.Installations[0].Source) + } +} diff --git a/cli/internal/agentpluginscli/source_transports_e2e_test.go b/cli/internal/agentpluginscli/source_transports_e2e_test.go new file mode 100644 index 000000000..c5f68d50f --- /dev/null +++ b/cli/internal/agentpluginscli/source_transports_e2e_test.go @@ -0,0 +1,273 @@ +package agentpluginscli + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "encoding/pem" + "io/fs" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/sourceacquisition" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" +) + +type directTransportFixture struct { + server *httptest.Server + work, remote string + mu sync.Mutex + payload []byte + downloads int +} + +func newDirectTransportFixture(t *testing.T, format string, private bool) *directTransportFixture { + t.Helper() + fixture := &directTransportFixture{work: dependencyRepo(t)} + project := t.TempDir() + fixture.remote = filepath.Join(project, "repo.git") + dependencyGit(t, fixture.work, "clone", "--quiet", "--bare", fixture.work, fixture.remote) + git, err := exec.LookPath("git") + if err != nil { + t.Fatal(err) + } + backend := &cgi.Handler{Path: git, Args: []string{"http-backend"}, Env: []string{"GIT_PROJECT_ROOT=" + project, "GIT_HTTP_EXPORT_ALL=1"}} + fixture.payload = directArchiveBytes(t, fixture.work, format, "provider-v1/") + fixture.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/repo.git/") { + if private { + user, password, ok := r.BasicAuth() + if !ok || user != "fixture" || password != "disposable-fixture-token" { + w.Header().Set("WWW-Authenticate", `Basic realm="fixture"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + } + backend.ServeHTTP(w, r) + return + } + fixture.mu.Lock() + defer fixture.mu.Unlock() + fixture.downloads++ + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(fixture.payload) + })) + t.Cleanup(fixture.server.Close) + caPath := filepath.Join(t.TempDir(), "ca.pem") + dependencyWrite(t, caPath, string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: fixture.server.Certificate().Raw}))) + t.Setenv("GIT_SSL_CAINFO", caPath) + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", home) + if private { + dependencyWrite(t, filepath.Join(home, ".gitconfig"), "[credential \""+fixture.server.URL+"\"]\n helper = \"!f() { printf 'username=fixture\\npassword=disposable-fixture-token\\n'; }; f\"\n") + } + return fixture +} + +func directArchiveBytes(t *testing.T, root, format, wrapper string) []byte { + t.Helper() + var result bytes.Buffer + if format == "git" || format == "private-git" { + return nil + } + if format == "zip" || format == "extensionless" { + writer := zip.NewWriter(&result) + directWalkPackage(t, root, func(name string, body []byte) error { + file, err := writer.Create(wrapper + name) + if err != nil { + return err + } + _, err = file.Write(body) + return err + }) + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return result.Bytes() + } + var target interface{ Write([]byte) (int, error) } = &result + var compressed *gzip.Writer + if format != "tar" { + compressed = gzip.NewWriter(&result) + target = compressed + } + writer := tar.NewWriter(target) + directWalkPackage(t, root, func(name string, body []byte) error { + if err := writer.WriteHeader(&tar.Header{Name: wrapper + name, Mode: 0600, Size: int64(len(body))}); err != nil { + return err + } + _, err := writer.Write(body) + return err + }) + if err := writer.Close(); err != nil { + t.Fatal(err) + } + if compressed != nil { + if err := compressed.Close(); err != nil { + t.Fatal(err) + } + } + return result.Bytes() +} + +func directWalkPackage(t *testing.T, root string, visit func(string, []byte) error) { + t.Helper() + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + if entry.Name() == ".git" { + return filepath.SkipDir + } + return nil + } + name, err := filepath.Rel(root, path) + if err != nil { + return err + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + return visit(filepath.ToSlash(name), body) + }) + if err != nil { + t.Fatal(err) + } +} + +func directCLIEnvironment(t *testing.T, transport *directTransportFixture) cliFixture { + t.Helper() + fixture, _, _ := dependencySpecHarness(t, transport.work, dependencyClientSpec{id: domain.ClientGemini, activation: "active"}) + fixture.app.SourceAcquirer = sourceacquisition.Acquirer{TempRoot: t.TempDir(), HTTPClient: transport.server.Client()} + return fixture +} + +func directLockedPin(t *testing.T, fixture cliFixture) environment.LockedPackage { + t.Helper() + manifestPath, lockPath := fixture.app.dependencyPaths() + requirements, err := readDependencyManifest(manifestPath) + if err != nil { + t.Fatal(err) + } + locked, err := readDependencyLock(lockPath, requirements) + if err != nil { + t.Fatal(err) + } + if len(locked.Packages) != 1 { + t.Fatalf("unexpected pins: %+v", locked.Packages) + } + return locked.Packages[0] +} + +// Git transports and archive downloads are native/real TLS. Agent process +// responses are inert fixtures; the standard loader, store, ownership and CLI +// transactions are production implementations. +func TestDirectSourceSavedLifecycleRealHTTPS(t *testing.T) { + for _, format := range []string{"git", "private-git", "zip", "tar", "tar.gz", "tgz", "extensionless"} { + t.Run(format, func(t *testing.T) { directSavedLifecycleCase(t, format) }) + } +} + +func directSavedLifecycleCase(t *testing.T, format string) { + t.Helper() + transport := newDirectTransportFixture(t, format, format == "private-git") + fixture := directCLIEnvironment(t, transport) + endpoint := transport.server.URL + "/package." + format + if format == "git" || format == "private-git" { + endpoint = transport.server.URL + "/repo.git" + } + if format == "extensionless" { + endpoint = transport.server.URL + "/download" + } + directSavedLifecycleEndpoint(t, fixture, transport, format, endpoint) +} + +func directSavedLifecycleEndpoint(t *testing.T, fixture cliFixture, transport *directTransportFixture, format, endpoint string) { + t.Helper() + dependencyRun(t, fixture, "add", endpoint, "--target", "gemini", "--format", "json") + first := directLockedPin(t, fixture) + state, err := fixture.store.Load() + if err != nil { + t.Fatal(err) + } + installationID, sourceID := state.Installations[0].InstallationID, state.Installations[0].Source.SourceBindingID + directAssertSourceFacts(t, first, state.Installations[0], endpoint) + directAdvanceTransport(t, transport, format) + dependencyRun(t, fixture, "update", first.Alias, "--target", "gemini", "--format", "json") + second := directLockedPin(t, fixture) + if second.TreeDigest == first.TreeDigest || second.SourceKey != first.SourceKey { + t.Fatal("moving source did not update within its stable ownership") + } + state, err = fixture.store.Load() + if err != nil { + t.Fatal(err) + } + if state.Installations[0].InstallationID != installationID || state.Installations[0].Source.SourceBindingID != sourceID { + t.Fatal("update changed binding identity") + } + dependencyRun(t, fixture, "repair", installationID, "--target", "gemini", "--format", "json") + directColdRestore(t, fixture, transport, second) +} + +func directAdvanceTransport(t *testing.T, transport *directTransportFixture, format string) { + t.Helper() + dependencyWrite(t, filepath.Join(transport.work, "revision.txt"), "second immutable package") + dependencyGit(t, transport.work, "add", ".") + dependencyGit(t, transport.work, "commit", "-m", "test: advance source") + dependencyGit(t, transport.work, "push", "--quiet", transport.remote, "main") + transport.mu.Lock() + defer transport.mu.Unlock() + transport.payload = directArchiveBytes(t, transport.work, format, "provider-v2/") +} + +func directAssertSourceFacts(t *testing.T, pin environment.LockedPackage, installation domain.Installation, endpoint string) { + t.Helper() + kind := "git" + if pin.Source.Archive != nil { + kind = "archive" + if pin.Source.Archive.SHA256 == "" || installation.Source.ArchiveSHA256 != pin.Source.Archive.SHA256 { + t.Fatal("archive byte checksum was lost") + } + } + if installation.Source.SourceKind != kind || installation.Source.SourceURL != endpoint || installation.Source.PackageSubpath != "" { + t.Fatalf("transport provenance lost: %+v", installation.Source) + } +} + +func directColdRestore(t *testing.T, original cliFixture, transport *directTransportFixture, pin environment.LockedPackage) { + t.Helper() + fresh := directCLIEnvironment(t, transport) + oldManifest, oldLock := original.app.dependencyPaths() + newManifest, newLock := fresh.app.dependencyPaths() + dependencyWrite(t, newManifest, dependencyRead(t, oldManifest)) + dependencyWrite(t, newLock, dependencyRead(t, oldLock)) + dependencyRun(t, fresh, "deps", "sync", "--locked", "--yes", "--format", "json") + state, err := fresh.store.Load() + if err != nil { + t.Fatal(err) + } + if len(state.Installations) != 1 || state.Installations[0].Source.TreeDigest != pin.TreeDigest || state.Installations[0].Source.ResolvedRevision != pin.Revision() { + t.Fatalf("cold restore changed pinned identity: %+v", state.Installations) + } + dependencyRun(t, fresh, "deps", "sync", "--locked", "--yes", "--format", "json") + view, err := fresh.app.Installer.Inspect(context.Background()) + if err != nil { + t.Fatal(err) + } + if view.Installations[0].SourceKey != pin.SourceKey { + t.Fatalf("inspection lost source key: %+v", view.Installations) + } +} diff --git a/docs/plans/source-transports-e2e/archive-code-review-r1.md b/docs/plans/source-transports-e2e/archive-code-review-r1.md new file mode 100644 index 000000000..0ead7f481 --- /dev/null +++ b/docs/plans/source-transports-e2e/archive-code-review-r1.md @@ -0,0 +1,30 @@ +# Archive implementation independent review R1 + +Reviewed 2026-10-23 by project_core, read-only, archive tree `6b8baf2f4849a19c33a1be24ac6a56b2c08aec33` (cumulative `fc6c4aee`, `2b267cc4`, `6b8baf2f`). Scope: new `adapters/sourceacquisition/archive*.go` and their reviewed source/redirect/dot-prefix contracts. No archive worktree edits. + +## Decision + +**Changes required: one P2 contract gap.** No containment, source-identity, redirect credential, or framing blocker found. Re-review the small fix below before R1 GO. Independent source lead R2 is still required at integration. + +## R1-1 — P2: enforce a recognized initial filename suffix against stream format + +`archive_download.go:archiveDownload` recognizes byte magic and sets explicit archive semantics but never compares the detected format with a recognized source suffix. `archiveCoordinatesFor`/`archiveExplicit` only use suffixes as a boolean. Consequently a ZIP payload fetched from `package.tar`, or an uncompressed TAR fetched from `package.tgz`, is accepted. The reviewed source plan explicitly requires detecting incompatible format/suffix (sources-plan.md archive policy). + +Add a small pure check after positive magic recognition against the **original normalized source endpoint path** (or explicit local filename): `.zip` -> ZIP; `.tar` -> TAR; `.tar.gz`/`.tgz` -> gzip TAR. Extensionless input remains magic-classified. A redirected CDN filename or Content-Type must remain nonauthoritative; keep all errors after positive recognition as archive errors, never `ErrNotArchive`. Test the mismatches, matching formats, and extensionless inputs without a second download. Existing byte-pin and framing gates remain in place. + +## Reviewed invariants + +- The one retained original stream is SHA256 hashed and compared with requested/locked byte pins before extraction. Tree pins are checked after selected-package resealing; mismatches clean the caller-owned sealed snapshot. +- Private extraction uses rooted exclusive writes; links/special files are never created. NFC portable lexical names, implicit ancestor collision tracking, case/Unicode folds, entry/depth/file/expanded limits and exactly one conventional `./` prefix preserve containment across ZIP and effective TAR/PAX names. +- Remote initial URLs remain query-free public HTTPS. Redirects remain bounded HTTPS, strip headers, do not inherit a cookie jar or redirect exception, keep signed query encoding transient, and expose sanitized errors. Original source coordinates alone populate source identity. +- ZIP audits ordinary single-disk central/local records, directory bounds, contiguous declared regions and exact end/comment boundaries. TAR preflight requires two zero end blocks and bounded aligned zero padding, rejects special/global/sparse entries and audits metadata counts/sizes. Gzip accepts exactly one CRC/ISIZE-validated member with no outer tail. +- Wrapper removal is transient and logical package paths are stable; malformed root manifests do not silently select nested packages. Candidate discovery is bounded and explicit selection remains available. +- Context/timeout applies through HTTP, retained reads, unpacking and resealing; failure cleanup owns the private workspace. Package code is not executed and the existing standard loader remains schema authority. + +## Verification + +Independent full `go test -p=1 ./install/integrationctl/agentplugins/adapters/sourceacquisition -count=1 -timeout=2m` passed in **2.005s**, Go 1.26.10, GOMAXPROCS=3, using the shared module/build caches. The first sandboxed attempt could not bind its loopback TLS listener; the same test invocation with authorized loopback execution passed. This includes real TLS redirects, ZIP/TAR/TGZ acquisition, byte/tree pins, hostile paths/types/collisions, strict archive endings, cancellation, real >256 MiB download/gzip bounds and cleanup. These are transport fixtures; no real-client certification claim is made. + +## Fix verification — R1 GO + +Reviewed fix `b99a15a497b1d0fb916a62965c0a33568ab9988f`: the recognized initial local/HTTPS suffix now matches detected magic before the retained stream is written. All ZIP/TAR/gzip mismatch combinations are refused without `ErrNotArchive`; `.tgz` and `.tar.gz` remain equivalent, extensionless classification is unchanged, and a signed redirect's `.tar` filename remains nonauthoritative for an initial `.zip` source. Independent focused real TLS suffix/redirect tests passed **0.139s**. R1-1 is closed; **R1 GO**, pending independent R2 at integration. diff --git a/docs/plans/source-transports-e2e/archive-code-review-r2.md b/docs/plans/source-transports-e2e/archive-code-review-r2.md new file mode 100644 index 000000000..472c05738 --- /dev/null +++ b/docs/plans/source-transports-e2e/archive-code-review-r2.md @@ -0,0 +1,26 @@ +# Archive implementation independent review R2 + +Reviewed 2026-10-23 by sources_lead. Scope: cumulative archive component through `b99a15a497b1d0fb916a62965c0a33568ab9988f`, including the two reviewed compatibility amendments and R1 suffix fix. The review reads production extraction, framing, transport, package selection and cleanup code independently; it does not reuse the archive author's conclusions. + +## Decision + +**R2 GO.** No remaining correctness, containment, provenance or format-policy blocker found. R1's initial suffix/magic mismatch finding is closed by the narrow original-endpoint check before stream publication; extensionless and signed redirect handling retain their intended behavior. + +## Reviewed boundaries + +- The retained stream is bounded, hashed exactly once before extraction, and checked against requested/locked original-byte SHA256. A recognized archive never falls back to Git after corruption, unsupported framing, selection or pin failures. HTTP status/nonarchive classification is separate from explicit archive intent. +- Initial intent is query-free HTTPS; transient redirect targets preserve bounded signed query encoding while rejecting userinfo, fragments, controls and downgrade. Every hop clears headers, client cookies/callbacks are excluded, errors omit transient URLs and bodies, and only the original endpoint is persisted. The default transport and injected ordinary TLS transport both enforce certificate verification. +- ZIP central/local/descriptor records, counts, method/flags, CRC/size checks and contiguous regions reject preambles, undeclared gaps, ZIP64, overlapping records and trailers. TAR audits raw record framing before standard decoding, including effective PAX size, metadata limits, two terminal zero blocks and aligned zero padding; gzip validates one member with no outer tail. +- Extraction rejects links/special files and uses exclusive rooted writes. Exactly one conventional `./` prefix is removed before NFC/portable path checks and collision tracking. Implicit directory, Unicode/case and file-versus-directory collisions share one policy. Entries, depth, file size, expanded bytes and raw/gzip stream expansion are bounded independently. +- A provider wrapper is transient. Logical selected paths exclude that wrapper, root manifest presence has precedence, explicit subpaths remain logical, and candidate diagnostics are bounded without another HTTP acquisition. The unchanged standard loader remains manifest authority; package code is never executed. +- Context, bounded timeout, private stream/extraction cleanup, snapshot cleanup on pin mismatch and caller ownership of the sealed snapshot remain explicit. + +## Verification + +The source integration additionally passed a real HTTPS CLI lifecycle matrix for public Git, credential-helper private Git, ZIP, TAR, TAR.GZ, TGZ and extensionless archives: saved add, stable-owner update across a changing archive wrapper, repair, cold locked restore and repeat sync. Native agent process responses are inert fixtures; this is source/lifecycle proof, not real agent certification. Full independent archive and native acquisition package tests are recorded below when completed. + +Independent complete source acquisition package tests passed in **6.107s**, Go 1.26.10/GOMAXPROCS=3, with native loopback execution. This includes the full hostile archive/format/stream-limit suite and actual Git HTTPS/private/SSH policy fixtures. R2 GO remains confirmed. + +## Repeated cancellation correction + +A later repeated run exposed a P2 cancellation return-shape race: a canceled HTTP body could end with clean EOF, after which framing validation reported truncated ZIP instead of context cancellation. R2 was narrowly reopened for this finding. Corrective archive commit `09131147` checks context unconditionally after retained/gzip copy and close, before unpacking, and during gzip expansion. The existing assertion is unchanged. Independent `TestArchiveCancellationAndDeclaredDownloadLimit -count=100` passes; **R2 GO restored** after this fix. diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive.go index 496d5a1ca..6c2ff2fef 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive.go @@ -118,6 +118,9 @@ func (acquirer Acquirer) archiveRetain(ctx context.Context, coordinates archiveC } func archiveUnpack(ctx context.Context, filename, extractionRoot string, format archiveFormat) ([]string, error) { + if err := ctx.Err(); err != nil { + return nil, err + } file, err := os.Open(filename) if err != nil { return nil, errors.New("open retained archive stream") diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_download.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_download.go index 102f5255f..a1641d340 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_download.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_download.go @@ -164,13 +164,13 @@ func archiveSave(ctx context.Context, reader io.Reader, destination string) (str hash := sha256.New() n, copyErr := io.Copy(io.MultiWriter(file, hash), io.LimitReader(reader, archiveMaxBytes+1)) closeErr := file.Close() + if err := ctx.Err(); err != nil { + return "", err + } if n > archiveMaxBytes { return "", errors.New("archive download exceeds 256 MiB") } if copyErr != nil || closeErr != nil { - if ctx.Err() != nil { - return "", ctx.Err() - } return "", errors.New("archive download was incomplete") } return hex.EncodeToString(hash.Sum(nil)), nil diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_tar.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_tar.go index d1e62dbec..c91294f6f 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_tar.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_tar.go @@ -34,8 +34,11 @@ func archiveGunzip(ctx context.Context, compressed, destination string) error { if err != nil { return err } - n, copyErr := io.Copy(output, io.LimitReader(reader, archiveMaxBytes+1)) + n, copyErr := io.Copy(output, io.LimitReader(&archiveContextReader{ctx: ctx, reader: reader}, archiveMaxBytes+1)) closeErr := output.Close() + if err := ctx.Err(); err != nil { + return err + } if n > archiveMaxBytes { return errors.New("gzip archive exceeds 256 MiB expanded stream limit") } diff --git a/install/integrationctl/agentplugins/adapters/statev2/store.go b/install/integrationctl/agentplugins/adapters/statev2/store.go index 3fb4e5c33..01850e1bc 100644 --- a/install/integrationctl/agentplugins/adapters/statev2/store.go +++ b/install/integrationctl/agentplugins/adapters/statev2/store.go @@ -14,6 +14,7 @@ import ( "github.com/777genius/plugin-kit-ai/install/integrationctl/adapters/atomicfile" "github.com/777genius/plugin-kit-ai/install/integrationctl/adapters/pathpolicy" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" ) const ( @@ -221,6 +222,9 @@ func Validate(state domain.StateFileV2) error { return fmt.Errorf("%s references unknown client binding", prefix) } sourceKey := "github:" + strings.ToLower(installation.Source.Repository) + ":" + installation.Source.PackageSubpath + if installation.Source.SourceKind != "" { + sourceKey = sourcespec.Key(sourcespec.Kind(installation.Source.SourceKind), installation.Source.SourceURL, installation.Source.PackageSubpath) + } if installation.Directory != nil { sourceKey = "directory:" + strings.ToLower(installation.Directory.DistributionID) } diff --git a/install/integrationctl/agentplugins/domain/identity.go b/install/integrationctl/agentplugins/domain/identity.go index 0b4c40343..6701171aa 100644 --- a/install/integrationctl/agentplugins/domain/identity.go +++ b/install/integrationctl/agentplugins/domain/identity.go @@ -26,7 +26,7 @@ func newInstallationID(reader io.Reader) (string, error) { func ComputeSourceBindingID(source SourceIdentity) string { canonicalSource := source.CanonicalSource - if source.SourceBindingHint == "uap-dependency-git" || source.SourceBindingHint == "uap-dependency-registry" { + if source.Repository != "" && (source.SourceBindingHint == "uap-dependency-git" || source.SourceBindingHint == "uap-dependency-registry") { canonicalSource = "github:" + strings.ToLower(source.Repository) + ":" + source.PackageSubpath } canonical := strings.Join([]string{ diff --git a/install/integrationctl/agentplugins/installer/acquired_source.go b/install/integrationctl/agentplugins/installer/acquired_source.go new file mode 100644 index 000000000..4759c5aa4 --- /dev/null +++ b/install/integrationctl/agentplugins/installer/acquired_source.go @@ -0,0 +1,61 @@ +package installer + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +func acquiredDirectSourceKey(source domain.SourceIdentity, portable bool) (string, error) { + var endpoint string + var err error + switch source.SourceKind { + case "git": + endpoint, _, err = sourcespec.NormalizeGitURL(source.SourceURL) + if !sourcespec.ValidCommit(source.ResolvedRevision) || source.ArchiveSHA256 != "" { + return "", ErrInvalidRequest + } + case "archive": + endpoint, err = sourcespec.NormalizeArchiveURL(source.SourceURL) + if !portable && filepath.IsAbs(source.SourceURL) { + endpoint, err = filepath.Clean(source.SourceURL), nil + } + if !sourcespec.ValidSHA256(source.ArchiveSHA256) || source.ResolvedRevision != "sha256:"+source.ArchiveSHA256 { + return "", ErrInvalidRequest + } + default: + return "", ErrInvalidRequest + } + if err != nil || endpoint != source.SourceURL || source.Repository != "" { + return "", ErrInvalidRequest + } + if _, err := sourcespec.NormalizeSubpath(source.PackageSubpath); err != nil { + return "", ErrInvalidRequest + } + key := sourcespec.Key(sourcespec.Kind(source.SourceKind), endpoint, source.PackageSubpath) + if key != source.CanonicalSource { + return "", fmt.Errorf("%w: acquired direct source identity differs", ErrInvalidRequest) + } + return key, nil +} + +func validateAcquiredSource(acquired *domain.AcquiredSnapshot, portable bool) (string, error) { + source := acquired.Source + if source.SourceKind != "" { + if acquired.OriginMode != domain.OriginModeDirect { + return "", ErrInvalidRequest + } + return acquiredDirectSourceKey(source, portable) + } + expected := "github:" + source.Repository + "@" + source.ResolvedRevision + if source.PackageSubpath != "" { + expected += "//" + source.PackageSubpath + } + if source.Repository == "" || !sourcespec.ValidCommit(source.ResolvedRevision) || (acquired.OriginMode == domain.OriginModeDirect && source.RequestedSource != expected) || source.CanonicalSource != expected { + return "", fmt.Errorf("%w: acquired source coordinates disagree", ErrInvalidRequest) + } + return "github:" + strings.ToLower(source.Repository) + ":" + source.PackageSubpath, nil +} diff --git a/install/integrationctl/agentplugins/installer/group_prepare.go b/install/integrationctl/agentplugins/installer/group_prepare.go index e1c37dce6..551c165e2 100644 --- a/install/integrationctl/agentplugins/installer/group_prepare.go +++ b/install/integrationctl/agentplugins/installer/group_prepare.go @@ -204,7 +204,9 @@ func (e *Engine) loadGroupSnapshot(ctx context.Context, handle *PreparedOperatio return domain.PackageEnvelope{}, "", err } } - e.reuseMatchingSourceIdentity(req.InstallationID, &snapshot, req.Operation == OpUpdate) + if req.AcquiredSnapshot == nil { + e.reuseMatchingSourceIdentity(req.InstallationID, &snapshot, req.Operation == OpUpdate) + } handle.snapshots[len(handle.snapshots)-1] = snapshot if i == 0 { handle.snapshot = snapshot diff --git a/install/integrationctl/agentplugins/installer/inspect.go b/install/integrationctl/agentplugins/installer/inspect.go index 5177a2eac..579ee4e12 100644 --- a/install/integrationctl/agentplugins/installer/inspect.go +++ b/install/integrationctl/agentplugins/installer/inspect.go @@ -16,6 +16,7 @@ import ( "github.com/777genius/plugin-kit-ai/install/integrationctl/adapters/dirswap" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/clients/opencode" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/transaction" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/usecase" ) @@ -116,6 +117,9 @@ func inspectInstallations(state domain.StateFileV2) ([]InspectedInstallation, ma TreeDigest: installation.Source.TreeDigest, } item.SourceKey = "github:" + strings.ToLower(installation.Source.Repository) + ":" + installation.Source.PackageSubpath + if installation.Source.SourceKind == "git" || installation.Source.SourceKind == "archive" { + item.SourceKey = sourcespec.Key(sourcespec.Kind(installation.Source.SourceKind), installation.Source.SourceURL, installation.Source.PackageSubpath) + } if installation.Directory != nil { item.SourceKey = "directory:" + strings.ToLower(installation.Directory.DistributionID) } diff --git a/install/integrationctl/agentplugins/installer/package.go b/install/integrationctl/agentplugins/installer/package.go index 37a46ede6..5000095b9 100644 --- a/install/integrationctl/agentplugins/installer/package.go +++ b/install/integrationctl/agentplugins/installer/package.go @@ -338,6 +338,9 @@ func (e *Engine) reuseMatchingSourceIdentity(installationID string, snapshot *do if installation.Source.CanonicalSource != "" { snapshot.Source.CanonicalSource = installation.Source.CanonicalSource } + snapshot.Source.SourceKind = installation.Source.SourceKind + snapshot.Source.SourceURL = installation.Source.SourceURL + snapshot.Source.ArchiveSHA256 = installation.Source.ArchiveSHA256 if installation.Source.RequestedSource != "" { snapshot.Source.RequestedSource = installation.Source.RequestedSource } @@ -374,16 +377,11 @@ func wrapLifecycleError(err error) error { func validateAcquiredRequest(req Request) error { acquired := req.AcquiredSnapshot - source := acquired.Source - expected := "github:" + source.Repository + "@" + source.ResolvedRevision - if source.PackageSubpath != "" { - expected += "//" + source.PackageSubpath - } - if source.Repository == "" || len(source.ResolvedRevision) != 40 || (acquired.OriginMode == domain.OriginModeDirect && source.RequestedSource != expected) || source.CanonicalSource != expected { - return fmt.Errorf("%w: acquired source coordinates disagree", ErrInvalidRequest) + key, err := validateAcquiredSource(acquired, req.EnvironmentAuthority != nil) + if err != nil { + return err } if req.EnvironmentAuthority != nil { - key := "github:" + strings.ToLower(acquired.Source.Repository) + ":" + acquired.Source.PackageSubpath if acquired.OriginMode == domain.OriginModeDirectory { if acquired.Directory == nil { return fmt.Errorf("%w: directory source is missing publication identity", ErrInvalidRequest) diff --git a/install/integrationctl/agentplugins/sourcespec/parse.go b/install/integrationctl/agentplugins/sourcespec/parse.go index 509708ae1..fbfa330a2 100644 --- a/install/integrationctl/agentplugins/sourcespec/parse.go +++ b/install/integrationctl/agentplugins/sourcespec/parse.go @@ -4,16 +4,23 @@ import ( "errors" "net/url" "strings" + "unicode" ) // Parse recognizes direct inputs only. Directory names and local directories // remain the caller's grammar; no filesystem existence probe is performed. func Parse(raw string) (Spec, error) { + if explicitArchivePath(raw) { + if len(raw) > 8192 || strings.TrimSpace(raw) != raw || strings.ContainsFunc(raw, unicode.IsControl) { + return Spec{}, errors.New("invalid local archive path") + } + return Spec{Requested: raw, Kind: KindLocalArchive, Archive: &Archive{URL: raw}}, nil + } if !safeText(raw) { return Spec{}, errors.New("invalid source input") } out := Spec{Requested: raw} - if !strings.Contains(raw, "://") && !strings.HasPrefix(raw, "github:") && !looksLikeSCP(raw) && HasArchiveSuffix(raw) { + if !strings.ContainsAny(raw, "/\\:") && HasArchiveSuffix(raw) { out.Kind, out.Archive = KindLocalArchive, &Archive{URL: raw} return out, nil } @@ -38,6 +45,13 @@ func Parse(raw string) (Spec, error) { return parseHTTPS(raw, out) } +func explicitArchivePath(raw string) bool { + if strings.Contains(raw, "://") || !HasArchiveSuffix(raw) { + return false + } + return strings.HasPrefix(raw, "./") || strings.HasPrefix(raw, "../") || strings.HasPrefix(raw, "/") || strings.ContainsRune(raw, '\\') || (len(raw) >= 3 && raw[1] == ':' && raw[2] == '/') +} + func looksLikeSCP(raw string) bool { if strings.Contains(raw, "://") { return false diff --git a/install/integrationctl/agentplugins/sourcespec/source_test.go b/install/integrationctl/agentplugins/sourcespec/source_test.go index 606dffdb5..2613e2a0b 100644 --- a/install/integrationctl/agentplugins/sourcespec/source_test.go +++ b/install/integrationctl/agentplugins/sourcespec/source_test.go @@ -34,6 +34,9 @@ func TestParseDirectSources(t *testing.T) { {"https://downloads.test/package#sha256=" + strings.Repeat("b", 64) + "&path=plugins%2Fdemo", KindArchive, nil, &Archive{URL: "https://downloads.test/package", SHA256: strings.Repeat("b", 64), Path: "plugins/demo"}}, {"https://example.test/package", KindHTTPSAuto, nil, &Archive{URL: "https://example.test/package"}}, {"./plugin.tgz", KindLocalArchive, nil, &Archive{URL: "./plugin.tgz"}}, + {"owner/repo@release.zip", KindGit, &Git{URL: "https://github.com/owner/repo.git", Repository: "owner/repo", Ref: "release.zip"}, nil}, + {`C:\tmp\package.zip`, KindLocalArchive, nil, &Archive{URL: `C:\tmp\package.zip`}}, + {"/tmp/package@1.0.zip", KindLocalArchive, nil, &Archive{URL: "/tmp/package@1.0.zip"}}, {"./package@1.0.zip", KindLocalArchive, nil, &Archive{URL: "./package@1.0.zip"}}, {"plugin.zip", KindLocalArchive, nil, &Archive{URL: "plugin.zip"}}, } diff --git a/install/integrationctl/agentplugins/usecase/binding.go b/install/integrationctl/agentplugins/usecase/binding.go index 376846eed..88edc78aa 100644 --- a/install/integrationctl/agentplugins/usecase/binding.go +++ b/install/integrationctl/agentplugins/usecase/binding.go @@ -269,6 +269,7 @@ func (service Service) commitBindingChange(state domain.StateFileV2, index int, func sourceBindingFromEnvelope(envelope domain.PackageEnvelope, sourceID string) domain.SourceBinding { return domain.SourceBinding{ SourceBindingID: sourceID, RequestedSource: envelope.Source.RequestedSource, + SourceKind: envelope.Source.SourceKind, SourceURL: envelope.Source.SourceURL, ArchiveSHA256: envelope.Source.ArchiveSHA256, CanonicalSource: envelope.Source.CanonicalSource, Repository: envelope.Source.Repository, PackageSubpath: envelope.Source.PackageSubpath, ResolvedRevision: envelope.Source.ResolvedRevision, TreeDigest: envelope.TreeDigest, diff --git a/install/integrationctl/agentplugins/usecase/group_commit.go b/install/integrationctl/agentplugins/usecase/group_commit.go index d036d85b0..4ef0433f2 100644 --- a/install/integrationctl/agentplugins/usecase/group_commit.go +++ b/install/integrationctl/agentplugins/usecase/group_commit.go @@ -82,12 +82,7 @@ func (session *groupSession) restoreGroupOrigin() { if session.input.Switch { installation := session.desired.Installations[session.installationIndex] first := session.first - installation.Source = domain.SourceBinding{ - SourceBindingID: session.sourceID, RequestedSource: first.Envelope.Source.RequestedSource, - CanonicalSource: first.Envelope.Source.CanonicalSource, Repository: first.Envelope.Source.Repository, - PackageSubpath: first.Envelope.Source.PackageSubpath, ResolvedRevision: first.Envelope.Source.ResolvedRevision, - TreeDigest: first.Envelope.TreeDigest, - } + installation.Source = sourceBindingFromEnvelope(first.Envelope, session.sourceID) installation.OriginMode = normalizedOriginMode(first.OriginMode) installation.Directory = cloneDirectoryOrigin(first.DirectoryResolution) session.desired.Installations[session.installationIndex] = installation diff --git a/install/integrationctl/agentplugins/usecase/installation.go b/install/integrationctl/agentplugins/usecase/installation.go index 4aa05b6ba..10259ab10 100644 --- a/install/integrationctl/agentplugins/usecase/installation.go +++ b/install/integrationctl/agentplugins/usecase/installation.go @@ -42,15 +42,7 @@ func newPreparedInstallation(input AddInput, installationID, sourceBindingID, ti return domain.Installation{ InstallationID: installationID, DeclaredName: input.Envelope.Manifest.Name, - Source: domain.SourceBinding{ - SourceBindingID: sourceBindingID, - RequestedSource: input.Envelope.Source.RequestedSource, - CanonicalSource: input.Envelope.Source.CanonicalSource, - Repository: input.Envelope.Source.Repository, - PackageSubpath: input.Envelope.Source.PackageSubpath, - ResolvedRevision: input.Envelope.Source.ResolvedRevision, - TreeDigest: input.Envelope.TreeDigest, - }, + Source: sourceBindingFromEnvelope(input.Envelope, sourceBindingID), Package: domain.PackageBinding{ LoaderKind: input.Envelope.LoaderKind, FormatID: input.Envelope.FormatID, SchemaURI: input.Envelope.SchemaURI, DeclaredName: input.Envelope.Manifest.Name, @@ -69,10 +61,9 @@ func overlayExistingInstallation(installation domain.Installation, input AddInpu installation.DeclaredName = input.Envelope.Manifest.Name installation.Source.ResolvedRevision = input.Envelope.Source.ResolvedRevision installation.Source.TreeDigest = input.Envelope.TreeDigest + installation.Source.ArchiveSHA256 = input.Envelope.Source.ArchiveSHA256 if installation.OriginMode == domain.OriginModeDirectory || input.EnvironmentAuthority != nil { - installation.Source = domain.SourceBinding{SourceBindingID: sourceBindingID, RequestedSource: input.Envelope.Source.RequestedSource, - CanonicalSource: input.Envelope.Source.CanonicalSource, Repository: input.Envelope.Source.Repository, PackageSubpath: input.Envelope.Source.PackageSubpath, - ResolvedRevision: input.Envelope.Source.ResolvedRevision, TreeDigest: input.Envelope.TreeDigest} + installation.Source = sourceBindingFromEnvelope(input.Envelope, sourceBindingID) } installation.Package = domain.PackageBinding{ LoaderKind: input.Envelope.LoaderKind, FormatID: input.Envelope.FormatID, From 5e12bd8f9236a4399489c3f1e766aed039d94241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Fri, 23 Oct 2026 19:38:37 +0300 Subject: [PATCH 07/25] fix(sources): repair exact direct revisions and retain operation cancellation --- README.md | 6 + cli/internal/agentpluginscli/add_multi.go | 14 +- .../agentpluginscli/deps_candidates.go | 2 +- .../agentpluginscli/deps_manifest_test.go | 2 +- cli/internal/agentpluginscli/deps_resolver.go | 16 +- cli/internal/agentpluginscli/lifecycle.go | 18 ++- cli/internal/agentpluginscli/read.go | 6 + cli/internal/agentpluginscli/repair_multi.go | 8 + cli/internal/agentpluginscli/saved_add.go | 2 +- cli/internal/agentpluginscli/saved_direct.go | 10 +- .../agentpluginscli/saved_lifecycle.go | 2 +- .../agentpluginscli/source_cancel_test.go | 59 +++++++ cli/internal/agentpluginscli/source_direct.go | 2 +- cli/internal/agentpluginscli/source_repair.go | 53 ++++++ .../agentpluginscli/source_repair_e2e_test.go | 137 ++++++++++++++++ .../source_selection_e2e_test.go | 102 ++++++++++++ .../agentpluginscli/source_ssh_e2e_test.go | 95 +++++++++++ docs/DIRECT_PACKAGE_SOURCES.md | 153 ++++++++++++++++++ docs/UAP_DEPENDENCIES.md | 27 +++- .../sources-code-review-r1.md | 23 +++ .../sources-code-review-r2.md | 67 ++++++++ .../sourceacquisition/archive_download.go | 2 +- .../sourceacquisition/git_transport.go | 2 +- .../agentplugins/sourcespec/parse.go | 22 ++- .../agentplugins/sourcespec/provider.go | 13 ++ .../agentplugins/sourcespec/source_test.go | 4 + 26 files changed, 825 insertions(+), 22 deletions(-) create mode 100644 cli/internal/agentpluginscli/source_cancel_test.go create mode 100644 cli/internal/agentpluginscli/source_repair.go create mode 100644 cli/internal/agentpluginscli/source_repair_e2e_test.go create mode 100644 cli/internal/agentpluginscli/source_selection_e2e_test.go create mode 100644 cli/internal/agentpluginscli/source_ssh_e2e_test.go create mode 100644 docs/DIRECT_PACKAGE_SOURCES.md create mode 100644 docs/plans/source-transports-e2e/sources-code-review-r2.md diff --git a/README.md b/README.md index a34634a96..dba5503db 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,12 @@ You can also install a local package or a pinned GitHub package without adding it to the registry. Direct-install examples are collected near the end of this README. +**SOURCE ONLY / UNRELEASED:** direct source support now includes arbitrary HTTPS +and SSH Git, GitHub/GitLab/Azure browser URLs, and complete HTTPS/local +ZIP/TAR/TAR.GZ/TGZ packages. See [Direct complete package sources](docs/DIRECT_PACKAGE_SOURCES.md) +for syntax, native credentials, immutable archive byte pins and supported limits. +Standalone skill files are not imported as plugins. + Other manifest formats are ignored and cannot override `plugin.json`. diff --git a/cli/internal/agentpluginscli/add_multi.go b/cli/internal/agentpluginscli/add_multi.go index b2ad800ad..419100ce6 100644 --- a/cli/internal/agentpluginscli/add_multi.go +++ b/cli/internal/agentpluginscli/add_multi.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "path/filepath" "strings" "unicode" "unicode/utf8" @@ -438,7 +439,7 @@ func newAddAcquisitionProof(loaded loadedPackage) (addAcquisitionProof, error) { ManifestDigest: loaded.envelope.ManifestDigest, ClosureDigest: groupedAcquisitionClosureDigest(sourceKind, loaded.envelope.Source, loaded.envelope.TreeDigest, loaded.envelope.ManifestDigest), SourceKind: sourceKind, - Fetched: loaded.envelope.Source.Repository != "" || loaded.envelope.Source.SourceKind != "", + Fetched: loaded.envelope.Source.Repository != "" || loaded.envelope.Source.SourceKind == "git" || loaded.envelope.Source.SourceKind == "archive" && !filepath.IsAbs(loaded.envelope.Source.SourceURL), Validated: true, }, nil } @@ -467,7 +468,9 @@ func acquisitionSourceKind(loaded loadedPackage) string { // are deliberately excluded because they may contain local, host-specific // paths. For local acquisitions, source kind plus the two validated package // identities form the closure; immutable remote acquisitions additionally bind -// repository, subpath, and revision. +// repository, subpath, and revision. New typed direct sources additionally bind +// the normalized endpoint and original archive byte checksum; legacy tuples stay +// byte-identical. func groupedAcquisitionClosureDigest(sourceKind string, source domain.SourceIdentity, treeDigest, manifestDigest string) string { hash := sha256.New() fields := []string{ @@ -479,6 +482,13 @@ func groupedAcquisitionClosureDigest(sourceKind string, source domain.SourceIden strings.TrimSpace(treeDigest), strings.TrimSpace(manifestDigest), } + if source.SourceKind != "" { + endpoint := source.SourceURL + if filepath.IsAbs(endpoint) { + endpoint = "" + } + fields = append(fields, endpoint, source.ArchiveSHA256) + } var size [8]byte for _, field := range fields { binary.BigEndian.PutUint64(size[:], uint64(len(field))) diff --git a/cli/internal/agentpluginscli/deps_candidates.go b/cli/internal/agentpluginscli/deps_candidates.go index 058622fcf..c34c6a93d 100644 --- a/cli/internal/agentpluginscli/deps_candidates.go +++ b/cli/internal/agentpluginscli/deps_candidates.go @@ -95,7 +95,7 @@ func (resolver dependencyResolver) AcquireCandidate(ctx context.Context, require return environment.ResolvedPackage{}, err } result := environment.ResolvedPackage{Package: pkg, Dependencies: facts} - return result, resolver.finishLoaded(loaded, pkg) + return result, resolver.finishLoaded(ctx, loaded, pkg) } func dependencyManifestAlias(name string) string { diff --git a/cli/internal/agentpluginscli/deps_manifest_test.go b/cli/internal/agentpluginscli/deps_manifest_test.go index 84614025a..ed5f967cb 100644 --- a/cli/internal/agentpluginscli/deps_manifest_test.go +++ b/cli/internal/agentpluginscli/deps_manifest_test.go @@ -171,7 +171,7 @@ func TestDependencyLockedShortNameDoesNotRecheckInstallEligibility(t *testing.T) _ = seed.cleanup() t.Fatal("fixture source does not match the recorded release digests") } - if err := (dependencyResolver{app: fixture.cli.app}).finishLoaded(seed); err != nil { + if err := (dependencyResolver{app: fixture.cli.app}).finishLoaded(context.Background(), seed); err != nil { t.Fatal(err) } requirements.Dependencies["docs"].Registry.Distribution = distribution.ProductID diff --git a/cli/internal/agentpluginscli/deps_resolver.go b/cli/internal/agentpluginscli/deps_resolver.go index 486aa230b..1feeb8a50 100644 --- a/cli/internal/agentpluginscli/deps_resolver.go +++ b/cli/internal/agentpluginscli/deps_resolver.go @@ -119,7 +119,7 @@ func (resolver dependencyResolver) resolveGit(ctx context.Context, requirement e if strings.Contains(locked.Alias, "/") || locked.Alias == "" { locked.Alias = dependencyManifestAlias(loaded.envelope.Manifest.Name) } - return locked, resolver.finishLoaded(loaded, locked) + return locked, resolver.finishLoaded(ctx, loaded, locked) } func (resolver dependencyResolver) resolveArchive(ctx context.Context, requirement environment.Requirement, targets []string) (environment.LockedPackage, error) { @@ -145,7 +145,7 @@ func (resolver dependencyResolver) resolveArchive(ctx context.Context, requireme _ = loaded.cleanup() return environment.LockedPackage{}, errors.New("archive requirement must explicitly select its logical package path") } - return locked, resolver.finishLoaded(loaded, locked) + return locked, resolver.finishLoaded(ctx, loaded, locked) } func (resolver dependencyResolver) resolveRegistry(ctx context.Context, requirement environment.Requirement, targets []string) (environment.LockedPackage, error) { @@ -170,7 +170,7 @@ func (resolver dependencyResolver) resolveRegistry(ctx context.Context, requirem if strings.Contains(locked.Alias, "/") || locked.Alias == "" { locked.Alias = dependencyManifestAlias(loaded.envelope.Manifest.Name) } - return locked, resolver.finishLoaded(loaded, locked) + return locked, resolver.finishLoaded(ctx, loaded, locked) } func registryLockedPackage(requirement environment.Requirement, loaded loadedPackage) (environment.LockedPackage, error) { @@ -191,14 +191,18 @@ func registryLockedPackage(requirement environment.Requirement, loaded loadedPac return locked, nil } -func (resolver dependencyResolver) finishLoaded(loaded loadedPackage, pins ...environment.LockedPackage) error { +func (resolver dependencyResolver) finishLoaded(ctx context.Context, loaded loadedPackage, pins ...environment.LockedPackage) error { + if err := ctx.Err(); err != nil { + _ = loaded.cleanup() + return err + } if _, err := packageDependencyFacts(loaded.envelope); err != nil { _ = loaded.cleanup() return err } if len(pins) > 0 && resolver.app.dependencySnapshots != nil { pin := pins[0] - snapshot, err := (packagedigest.Builder{}).SnapshotWithExecutables(context.Background(), loaded.envelope.SnapshotRoot, identityFromLocked(pin), loaded.envelope.ExecutableFiles) + snapshot, err := (packagedigest.Builder{}).SnapshotWithExecutables(ctx, loaded.envelope.SnapshotRoot, identityFromLocked(pin), loaded.envelope.ExecutableFiles) if err != nil { _ = loaded.cleanup() return err @@ -215,7 +219,7 @@ func (resolver dependencyResolver) finishLoaded(loaded loadedPackage, pins ...en snapshot := domain.PackageSnapshot{Root: loaded.envelope.SnapshotRoot, TreeDigest: loaded.envelope.TreeDigest, DigestAlgorithm: domain.TreeDigestAlgorithm, ExecutableFiles: append([]string{}, loaded.envelope.ExecutableFiles...), Source: loaded.envelope.Source} - if _, err := resolver.app.dependencyStore().Put(context.Background(), snapshot); err != nil { + if _, err := resolver.app.dependencyStore().Put(ctx, snapshot); err != nil { _ = loaded.cleanup() return err } diff --git a/cli/internal/agentpluginscli/lifecycle.go b/cli/internal/agentpluginscli/lifecycle.go index 533a23c42..729fa63f6 100644 --- a/cli/internal/agentpluginscli/lifecycle.go +++ b/cli/internal/agentpluginscli/lifecycle.go @@ -128,7 +128,12 @@ func runRepair(ctx context.Context, cmd *cobra.Command, app App, opts *options, return err } writeProgress(app, opts.format, "Resolving and validating the exact installed Agent Plugin revision...") - loaded, err := app.loadPackage(ctx, source) + var loaded loadedPackage + if installation.Source.SourceKind != "" { + loaded, err = app.loadRecordedDirect(ctx, installation, binding) + } else { + loaded, err = app.loadPackage(ctx, source) + } if err != nil { return err } @@ -693,6 +698,9 @@ func environmentAuthorityForBinding(state domain.StateFileV2, installationID str } func updateSource(installation domain.Installation) string { + if installation.Source.SourceKind == "archive" && filepath.IsAbs(installation.Source.SourceURL) { + return localArchiveUpdateSource(installation.Source) + } if installation.Source.SourceKind != "" && installation.Source.RequestedSource == "" { return directSourceRequested(installation.Source) } @@ -737,6 +745,14 @@ func repairSource(installation domain.Installation, binding domain.ClientBinding if revision == "" { return "", fmt.Errorf("the selected client binding has no exact resolved revision; refusing repair from a moving source") } + if installation.Source.SourceKind != "" { + source := installation.Source + source.ResolvedRevision = revision + if source.SourceKind == "archive" { + source.ArchiveSHA256 = strings.TrimPrefix(revision, "sha256:") + } + return directSourceRequested(source), nil + } if repository := strings.TrimSpace(installation.Source.Repository); repository != "" { value := "github:" + repository + "@" + revision if subpath := strings.TrimSpace(installation.Source.PackageSubpath); subpath != "" { diff --git a/cli/internal/agentpluginscli/read.go b/cli/internal/agentpluginscli/read.go index 174cefb39..9b40e16c1 100644 --- a/cli/internal/agentpluginscli/read.go +++ b/cli/internal/agentpluginscli/read.go @@ -584,6 +584,12 @@ func publicSource(source domain.SourceBinding) string { return publicRepositoryPath(source.Repository, source.PackageSubpath) } canonical := strings.TrimSpace(source.CanonicalSource) + if source.SourceKind != "" { + canonical = source.SourceURL + if source.SourceKind == "git" && !strings.Contains(canonical, "://") { + return canonical + } + } if canonical == "" || filepath.IsAbs(canonical) { return "local" } diff --git a/cli/internal/agentpluginscli/repair_multi.go b/cli/internal/agentpluginscli/repair_multi.go index 53f304e23..d161cf548 100644 --- a/cli/internal/agentpluginscli/repair_multi.go +++ b/cli/internal/agentpluginscli/repair_multi.go @@ -34,6 +34,7 @@ type repairMultiResult struct { } type repairRevisionRequest struct { + binding *domain.ClientBinding sequence uint64 revision string source string @@ -146,6 +147,10 @@ func (session *repairManySession) revisionKey(binding domain.ClientBinding) (str return "", request, err } request.source = source + if session.installation.Source.SourceKind != "" { + cloned := binding + request.binding = &cloned + } key = "direct:" + source } key += ":" + binding.PackageRevision.ResolvedRevision + ":" + binding.PackageRevision.TreeDigest + ":" + binding.PackageRevision.ManifestDigest @@ -193,6 +198,9 @@ func (session *repairManySession) resolveRevision(key string) (loadedPackage, er if session.installation.OriginMode == domain.OriginModeDirectory { return session.app.loadInstalledPackage(resolutionCtx, session.installation, request.targets, domain.DirectoryRepair, request.sequence, request.revision, session.detected) } + if request.binding != nil { + return session.app.loadRecordedDirect(resolutionCtx, session.installation, *request.binding) + } return session.app.loadPackageFor(resolutionCtx, request.source, packageResolutionRequest{ Targets: request.targets, Operation: domain.DirectoryRepair, Clients: session.detected, }) diff --git a/cli/internal/agentpluginscli/saved_add.go b/cli/internal/agentpluginscli/saved_add.go index d11be1215..bd439af62 100644 --- a/cli/internal/agentpluginscli/saved_add.go +++ b/cli/internal/agentpluginscli/saved_add.go @@ -167,7 +167,7 @@ func acquireSavedAdd(cmd *cobra.Command, app App, opts *options, source string, pin.ManifestDigest = loaded.envelope.ManifestDigest pin.Version = loaded.envelope.Manifest.Version pin.ExecutableFiles = append([]string{}, loaded.envelope.ExecutableFiles...) - if err := (dependencyResolver{app: app, dryRun: opts.dryRun}).finishLoaded(loaded, pin); err != nil { + if err := (dependencyResolver{app: app, dryRun: opts.dryRun}).finishLoaded(cmd.Context(), loaded, pin); err != nil { return environment.Requirement{}, environment.LockedPackage{}, err } return requirement, pin, nil diff --git a/cli/internal/agentpluginscli/saved_direct.go b/cli/internal/agentpluginscli/saved_direct.go index a9504d825..a2e92fbff 100644 --- a/cli/internal/agentpluginscli/saved_direct.go +++ b/cli/internal/agentpluginscli/saved_direct.go @@ -95,7 +95,7 @@ func resolveDirectSavedAdd(cmd *cobra.Command, app App, opts *options, source st } result.requirement = requirement pin = completeDirectPin(result, alias) - if err := (dependencyResolver{app: app, dryRun: opts.dryRun}).finishLoaded(result.loaded, pin); err != nil { + if err := (dependencyResolver{app: app, dryRun: opts.dryRun}).finishLoaded(cmd.Context(), result.loaded, pin); err != nil { return requirement, pin, err } return requirement, pin, nil @@ -140,3 +140,11 @@ func archivePathSelector(packagePath string) string { // Archive paths are already portable; the parser decodes fragment values. return "&path=" + url.QueryEscape(packagePath) } + +func localArchiveUpdateSource(source domain.SourceBinding) string { + spec, err := sourcespec.Parse(source.RequestedSource) + if err == nil && spec.Archive != nil && spec.Archive.SHA256 != "" { + return source.SourceURL + "#sha256=" + spec.Archive.SHA256 + archivePathSelector(source.PackageSubpath) + } + return source.SourceURL +} diff --git a/cli/internal/agentpluginscli/saved_lifecycle.go b/cli/internal/agentpluginscli/saved_lifecycle.go index 161359ae8..20e7f5765 100644 --- a/cli/internal/agentpluginscli/saved_lifecycle.go +++ b/cli/internal/agentpluginscli/saved_lifecycle.go @@ -356,7 +356,7 @@ func updateSavedRegistry(cmd *cobra.Command, app App, opts *options, snapshot en } requirements, candidate := savedRegistryCandidate(snapshot, alias, old, loaded) next, _ := candidate.Package(alias) - if err := (dependencyResolver{app: app, dryRun: opts.dryRun}).finishLoaded(loaded, next); err != nil { + if err := (dependencyResolver{app: app, dryRun: opts.dryRun}).finishLoaded(cmd.Context(), loaded, next); err != nil { return requirements, snapshot.Lock, err } // Validate the authoritative old graph before rebuilding edges from the new diff --git a/cli/internal/agentpluginscli/source_cancel_test.go b/cli/internal/agentpluginscli/source_cancel_test.go new file mode 100644 index 000000000..96e61786d --- /dev/null +++ b/cli/internal/agentpluginscli/source_cancel_test.go @@ -0,0 +1,59 @@ +package agentpluginscli + +import ( + "context" + "errors" + "os" + "strings" + "sync/atomic" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" +) + +type cancellationDuringRetention struct { + context.Context + cancel context.CancelFunc + checks atomic.Int32 +} + +func (ctx *cancellationDuringRetention) Err() error { + if ctx.checks.Add(1) == 2 { + ctx.cancel() + } + return ctx.Context.Err() +} + +func TestDependencyRetentionHonorsOperationCancellation(t *testing.T) { + fixture := newCLIFixture(t, nil) + loaded, err := fixture.app.acquireLocal(t.Context(), writeCLIPlugin(t)) + if err != nil { + t.Fatal(err) + } + originalRoot := loaded.envelope.SnapshotRoot + preview, cleanup, err := dependencyPreviewApp(fixture.app, &options{}) + if err != nil { + t.Fatal(err) + } + defer cleanup() + pin := environment.LockedPackage{Alias: "demo", SourceKey: "github:acme/demo:", DigestAlgorithm: domain.TreeDigestAlgorithm, + TreeDigest: loaded.envelope.TreeDigest, ManifestDigest: loaded.envelope.ManifestDigest, + Source: environment.LockedSource{Git: &environment.LockedGitSource{Repository: "acme/demo", Commit: strings.Repeat("a", 40)}}} + base, cancel := context.WithCancel(t.Context()) + defer cancel() + operation := &cancellationDuringRetention{Context: base, cancel: cancel} + err = (dependencyResolver{app: preview}).finishLoaded(operation, loaded, pin) + if !errors.Is(err, context.Canceled) { + t.Fatalf("retention detached operation cancellation: %v", err) + } + if len(preview.dependencySnapshots.snapshots) != 0 { + t.Fatal("canceled retention published a command snapshot") + } + if _, err := os.Stat(originalRoot); !os.IsNotExist(err) { + t.Fatal("canceled retention left original snapshot") + } + if _, err := os.Stat(preview.dependencyStore().Root); !os.IsNotExist(err) { + t.Fatal("canceled retention published durable store") + } +} diff --git a/cli/internal/agentpluginscli/source_direct.go b/cli/internal/agentpluginscli/source_direct.go index b38e2a0a3..fee88105f 100644 --- a/cli/internal/agentpluginscli/source_direct.go +++ b/cli/internal/agentpluginscli/source_direct.go @@ -25,7 +25,7 @@ func localArchiveInput(raw string) bool { if err != nil || spec.Kind != sourcespec.KindLocalArchive { return false } - info, err := os.Stat(raw) + info, err := os.Stat(spec.Archive.URL) return err != nil || !info.IsDir() } diff --git a/cli/internal/agentpluginscli/source_repair.go b/cli/internal/agentpluginscli/source_repair.go new file mode 100644 index 000000000..a741208af --- /dev/null +++ b/cli/internal/agentpluginscli/source_repair.go @@ -0,0 +1,53 @@ +package agentpluginscli + +import ( + "context" + "errors" + "strings" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +func (app App) loadRecordedDirect(ctx context.Context, installation domain.Installation, binding domain.ClientBinding) (loadedPackage, error) { + source := installation.Source + revision := binding.PackageRevision + if revision == nil || revision.TreeDigest == "" || revision.ManifestDigest == "" { + return loadedPackage{}, errors.New("recorded direct package revision is incomplete") + } + direct, err := app.directAcquirer() + if err != nil { + return loadedPackage{}, err + } + var snapshot domain.PackageSnapshot + switch source.SourceKind { + case "git": + snapshot, err = direct.AcquireGit(ctx, sourcespec.Git{URL: source.SourceURL, Path: source.PackageSubpath}, revision.ResolvedRevision, revision.TreeDigest) + case "archive": + checksum := strings.TrimPrefix(revision.ResolvedRevision, "sha256:") + if !sourcespec.ValidSHA256(checksum) || revision.ResolvedRevision != "sha256:"+checksum { + return loadedPackage{}, errors.New("recorded archive byte revision is invalid") + } + snapshot, err = direct.AcquireArchive(ctx, sourcespec.Archive{URL: source.SourceURL, Path: source.PackageSubpath}, checksum, revision.TreeDigest) + default: + return loadedPackage{}, errors.New("recorded source is not a typed direct package") + } + if err != nil { + return loadedPackage{}, err + } + if snapshot.Source.CanonicalSource != source.CanonicalSource { + _ = packagedigest.Remove(snapshot) + return loadedPackage{}, errors.New("recorded direct source identity changed") + } + snapshot.Source.RequestedSource = source.RequestedSource + loaded, err := app.loadStandardSnapshot(ctx, snapshot, domain.OriginModeDirect, nil, nil) + if err != nil { + return loadedPackage{}, err + } + if loaded.envelope.ManifestDigest != revision.ManifestDigest || loaded.envelope.TreeDigest != revision.TreeDigest { + _ = loaded.cleanup() + return loadedPackage{}, errors.New("recorded direct package digests disagree") + } + return loaded, nil +} diff --git a/cli/internal/agentpluginscli/source_repair_e2e_test.go b/cli/internal/agentpluginscli/source_repair_e2e_test.go new file mode 100644 index 000000000..e53e32ace --- /dev/null +++ b/cli/internal/agentpluginscli/source_repair_e2e_test.go @@ -0,0 +1,137 @@ +package agentpluginscli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/nativeconfig" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/providers" +) + +func TestDirectImperativeRepairUsesRecordedBytes(t *testing.T) { + for _, kind := range []string{"git", "zip", "local-archive"} { + t.Run(kind, func(t *testing.T) { directImperativeRepairCase(t, kind) }) + } +} + +func directImperativeRepairCase(t *testing.T, kind string) { + t.Helper() + format := kind + if kind == "local-archive" { + format = "zip" + } + transport := newDirectTransportFixture(t, format, false) + fixture := directCLIEnvironment(t, transport) + kernel := nativeconfig.New() + fixture.app.Lifecycle.NativeObserver = providers.NativeIdentityObserver{Stager: fixture.app.Lifecycle.Stager, Registry: fixture.app.ClientRegistry, NativeConfig: &kernel} + originalBytes := append([]byte{}, transport.payload...) + endpoint := transport.server.URL + "/package.zip" + if kind == "git" { + endpoint = transport.server.URL + "/repo.git" + } + if kind == "local-archive" { + endpoint = filepath.Join(t.TempDir(), "package@1.0.zip") + dependencyWrite(t, endpoint, string(transport.payload)) + } + dependencyRun(t, fixture, "add", endpoint, "--no-save", "--target", "gemini", "--format", "json") + before, err := fixture.store.Load() + if err != nil { + t.Fatal(err) + } + installation := before.Installations[0] + var target string + for _, binding := range installation.Clients { + target = binding.TargetLocator + } + targetManifest := filepath.Join(target, "plugin.json") + manifestBefore := dependencyRead(t, targetManifest) + directAdvanceTransport(t, transport, format) + if kind != "git" { + if kind == "local-archive" { + dependencyWrite(t, endpoint, string(transport.payload)) + } + stateBefore := dependencyRead(t, fixture.store.Path) + if _, _, err := fixture.execute(false, "repair", installation.InstallationID, "--target", "gemini", "--format", "json"); err == nil || !strings.Contains(err.Error(), "SHA256") { + t.Fatalf("changed archive bytes did not fail closed: %v", err) + } + if dependencyRead(t, fixture.store.Path) != stateBefore || dependencyRead(t, targetManifest) != manifestBefore { + t.Fatal("failed immutable repair changed state or target") + } + if kind == "local-archive" { + dependencyWrite(t, endpoint, string(originalBytes)) + } else { + transport.mu.Lock() + transport.payload = originalBytes + transport.mu.Unlock() + } + + } + dependencyRun(t, fixture, "repair", installation.InstallationID, "--target", "gemini", "--format", "json") + after, err := fixture.store.Load() + if err != nil { + t.Fatal(err) + } + if after.Installations[0].Source.ResolvedRevision != installation.Source.ResolvedRevision || after.Installations[0].Source.TreeDigest != installation.Source.TreeDigest { + t.Fatal("repair advanced to the moving source") + } + if _, err := os.Stat(filepath.Join(target, "revision.txt")); !os.IsNotExist(err) { + t.Fatal("repair copied newer source bytes") + } +} + +func TestSavedRepairRetainsOldPinAfterSourceAdvance(t *testing.T) { + for _, format := range []string{"git", "zip"} { + t.Run(format, func(t *testing.T) { + transport := newDirectTransportFixture(t, format, false) + fixture := directCLIEnvironment(t, transport) + endpoint := transport.server.URL + "/package.zip" + if format == "git" { + endpoint = transport.server.URL + "/repo.git" + } + dependencyRun(t, fixture, "add", endpoint, "--target", "gemini", "--format", "json") + pin := directLockedPin(t, fixture) + state, err := fixture.store.Load() + if err != nil { + t.Fatal(err) + } + installation := state.Installations[0] + var target string + for _, binding := range installation.Clients { + target = binding.TargetLocator + } + dependencyWrite(t, filepath.Join(target, "drift.txt"), "drift") + directAdvanceTransport(t, transport, format) + dependencyRun(t, fixture, "repair", installation.InstallationID, "--target", "gemini", "--format", "json") + after := directLockedPin(t, fixture) + if after.Revision() != pin.Revision() || after.TreeDigest != pin.TreeDigest { + t.Fatal("repair changed saved pin") + } + if format == "zip" { + directColdArchiveRefusal(t, fixture, transport) + } + }) + } +} + +func directColdArchiveRefusal(t *testing.T, original cliFixture, transport *directTransportFixture) { + t.Helper() + fresh := directCLIEnvironment(t, transport) + oldManifest, oldLock := original.app.dependencyPaths() + manifest, lock := fresh.app.dependencyPaths() + dependencyWrite(t, manifest, dependencyRead(t, oldManifest)) + dependencyWrite(t, lock, dependencyRead(t, oldLock)) + beforeManifest, beforeLock := dependencyRead(t, manifest), dependencyRead(t, lock) + if _, _, err := fresh.execute(false, "deps", "sync", "--locked", "--yes", "--format", "json"); err == nil || !strings.Contains(err.Error(), "SHA256") { + t.Fatalf("cold restore accepted changed archive bytes: %v", err) + } + if dependencyRead(t, manifest) != beforeManifest || dependencyRead(t, lock) != beforeLock { + t.Fatal("failed cold restore changed portable agreement") + } + for _, path := range []string{fresh.app.EnvironmentStoreRoot, fresh.store.Path} { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("failed restore published %s", filepath.Base(path)) + } + } +} diff --git a/cli/internal/agentpluginscli/source_selection_e2e_test.go b/cli/internal/agentpluginscli/source_selection_e2e_test.go new file mode 100644 index 000000000..d2e4da52c --- /dev/null +++ b/cli/internal/agentpluginscli/source_selection_e2e_test.go @@ -0,0 +1,102 @@ +package agentpluginscli + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func directPackageLayout(t *testing.T, paths ...string) string { + t.Helper() + source := writeCLIPlugin(t) + writeCLIMCP(t, source) + root := t.TempDir() + for _, packagePath := range paths { + directWalkPackage(t, source, func(name string, body []byte) error { + target := filepath.Join(root, packagePath, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(target), 0700); err != nil { + return err + } + return os.WriteFile(target, body, 0600) + }) + } + return root +} + +func TestDirectArchiveSelectionUsesOneDownloadAndLogicalPath(t *testing.T) { + for _, count := range []int{1, 2} { + t.Run(strings.Repeat("package", count), func(t *testing.T) { + transport := newDirectTransportFixture(t, "zip", false) + fixture := directCLIEnvironment(t, transport) + paths := []string{"plugins/demo"} + if count == 2 { + paths = append(paths, "plugins/other") + } + transport.payload = directArchiveBytes(t, directPackageLayout(t, paths...), "zip", "changing-provider-wrapper/") + endpoint := transport.server.URL + "/packages.zip" + _, _, err := fixture.execute(false, "add", endpoint, "--target", "gemini", "--format", "json") + if transport.downloads != 1 { + t.Fatalf("selection downloaded source %d times", transport.downloads) + } + if count == 2 { + if err == nil || !strings.Contains(err.Error(), "multiple packages") { + t.Fatalf("multiple candidates were not refused: %v", err) + } + if _, err := os.Stat(fixture.app.EnvironmentStoreRoot); !os.IsNotExist(err) { + t.Fatal("failed selection published store") + } + return + } + if err != nil { + t.Fatal(err) + } + pin := directLockedPin(t, fixture) + if pin.Source.Archive.Path != "plugins/demo" { + t.Fatalf("provider wrapper entered logical path: %+v", pin.Source.Archive) + } + transport.mu.Lock() + transport.payload = directArchiveBytes(t, directPackageLayout(t, paths...), "zip", "next-provider-wrapper/") + transport.mu.Unlock() + dependencyRun(t, fixture, "update", pin.Alias, "--target", "gemini", "--format", "json") + after := directLockedPin(t, fixture) + if after.SourceKey != pin.SourceKey || after.Source.Archive.Path != pin.Source.Archive.Path { + t.Fatal("wrapper update changed selected source identity") + } + }) + } +} + +func TestDirectRecognizedInvalidPackagesNeverInstallOrFallback(t *testing.T) { + for _, failure := range []string{"corrupt-zip", "raw-skill", "malformed-root"} { + t.Run(failure, func(t *testing.T) { + transport := newDirectTransportFixture(t, "zip", false) + fixture := directCLIEnvironment(t, transport) + switch failure { + case "corrupt-zip": + transport.payload = []byte("PK\x03\x04corrupt archive") + case "raw-skill": + root := t.TempDir() + dependencyWrite(t, filepath.Join(root, "SKILL.md"), "---\nname: raw\ndescription: raw skill\n---\nInstructions") + transport.payload = directArchiveBytes(t, root, "zip", "") + case "malformed-root": + root := directPackageLayout(t, "nested/demo") + dependencyWrite(t, filepath.Join(root, "plugin.json"), "{malformed") + transport.payload = directArchiveBytes(t, root, "zip", "") + } + _, _, err := fixture.execute(false, "add", transport.server.URL+"/package.zip", "--target", "gemini", "--format", "json") + if err == nil { + t.Fatal("invalid complete-package source installed") + } + if transport.downloads != 1 { + t.Fatalf("recognized archive retried as Git: %d requests", transport.downloads) + } + manifest, lock := fixture.app.dependencyPaths() + for _, path := range []string{manifest, lock, fixture.app.EnvironmentStoreRoot, fixture.store.Path} { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("invalid source published %s", filepath.Base(path)) + } + } + }) + } +} diff --git a/cli/internal/agentpluginscli/source_ssh_e2e_test.go b/cli/internal/agentpluginscli/source_ssh_e2e_test.go new file mode 100644 index 000000000..521623ece --- /dev/null +++ b/cli/internal/agentpluginscli/source_ssh_e2e_test.go @@ -0,0 +1,95 @@ +package agentpluginscli + +import ( + "fmt" + "net" + "os" + "os/exec" + "os/user" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestDirectSourceSavedLifecycleRealSSH(t *testing.T) { + sshd, err := exec.LookPath("sshd") + if err != nil { + t.Skip("real SSH CLI fixture requires openssh-server") + } + current, err := user.Current() + if err != nil { + t.Fatal(err) + } + for _, spelling := range []string{"ssh-url", "scp"} { + t.Run(spelling, func(t *testing.T) { + transport := newDirectTransportFixture(t, "git", false) + port, config := directSSHServer(t, sshd, transport.remote, current.Username) + t.Setenv("GIT_SSH_COMMAND", "ssh -F "+config) + endpoint := fmt.Sprintf("ssh://%s@127.0.0.1:%d%s", current.Username, port, transport.remote) + if spelling == "scp" { + endpoint = current.Username + "@uap-cli-fixture:" + transport.remote + } + fixture := directCLIEnvironment(t, transport) + directSavedLifecycleEndpoint(t, fixture, transport, "git", endpoint) + }) + } +} + +func directSSHServer(t *testing.T, sshd, repository, username string) (int, string) { + t.Helper() + root := t.TempDir() + for _, name := range []string{"host", "identity"} { + command := exec.CommandContext(t.Context(), "ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", filepath.Join(root, name)) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("generate disposable SSH key: %v %s", err, output) + } + } + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := listener.Addr().(*net.TCPAddr).Port + if err := listener.Close(); err != nil { + t.Fatal(err) + } + git, err := exec.LookPath("git") + if err != nil { + t.Fatal(err) + } + public := dependencyRead(t, filepath.Join(root, "identity.pub")) + dependencyWrite(t, filepath.Join(root, "authorized_keys"), fmt.Sprintf("command=\"%s upload-pack '%s'\",no-port-forwarding,no-agent-forwarding,no-X11-forwarding,no-pty %s", git, repository, public)) + serverConfig := filepath.Join(root, "sshd_config") + dependencyWrite(t, serverConfig, fmt.Sprintf("Port %d\nListenAddress 127.0.0.1\nHostKey %s\nPidFile %s\nAuthorizedKeysFile %s\nStrictModes no\nPasswordAuthentication no\nKbdInteractiveAuthentication no\nUsePAM no\nPermitRootLogin forced-commands-only\nAllowUsers %s\nLogLevel ERROR\n", port, filepath.Join(root, "host"), filepath.Join(root, "pid"), filepath.Join(root, "authorized_keys"), username)) + logs, err := os.Create(filepath.Join(root, "sshd.log")) + if err != nil { + t.Fatal(err) + } + command := exec.CommandContext(t.Context(), sshd, "-D", "-e", "-f", serverConfig) + command.Stdout, command.Stderr = logs, logs + if err := command.Start(); err != nil { + _ = logs.Close() + t.Fatal(err) + } + t.Cleanup(func() { _ = command.Process.Kill(); _ = command.Wait(); _ = logs.Close() }) + directWaitSSH(t, port, logs.Name()) + host := strings.Fields(dependencyRead(t, filepath.Join(root, "host.pub"))) + dependencyWrite(t, filepath.Join(root, "known_hosts"), fmt.Sprintf("[127.0.0.1]:%d %s %s\n", port, host[0], host[1])) + clientConfig := filepath.Join(root, "ssh_config") + dependencyWrite(t, clientConfig, fmt.Sprintf("Host *\n HostName 127.0.0.1\n Port %d\n User %s\n IdentityFile %s\n IdentitiesOnly yes\n UserKnownHostsFile %s\n StrictHostKeyChecking yes\n BatchMode yes\n", port, username, filepath.Join(root, "identity"), filepath.Join(root, "known_hosts"))) + return port, clientConfig +} + +func directWaitSSH(t *testing.T, port int, log string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + connection, err := (&net.Dialer{Timeout: 100 * time.Millisecond}).DialContext(t.Context(), "tcp", fmt.Sprintf("127.0.0.1:%d", port)) + if err == nil { + _ = connection.Close() + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("disposable SSH server failed: %s", dependencyRead(t, log)) +} diff --git a/docs/DIRECT_PACKAGE_SOURCES.md b/docs/DIRECT_PACKAGE_SOURCES.md new file mode 100644 index 000000000..aab7323f0 --- /dev/null +++ b/docs/DIRECT_PACKAGE_SOURCES.md @@ -0,0 +1,153 @@ +# Direct complete package sources + +**SOURCE ONLY / UNRELEASED.** These transports are implemented on the source +feature branch. They are not a claim that a published installer already includes +this behavior, or that every client supports every package component. + +A direct source must contain a complete standard Agent Plugins package with +`plugin.json`. Its skills, MCP configuration and other components use the existing +standard loader and target compatibility/security checks. A standalone `SKILL.md` +is not imported as a plugin, and UAP does not crawl an HTTPS directory listing. + +## Accepted inputs + +```bash +# Arbitrary HTTPS Git, tracking the remote's default branch +agentplugins add 'https://git.example.com/team/tools.git' --target gemini + +# Explicit branch/tag, or an immutable full lowercase 40-character Git commit +agentplugins add 'https://git.example.com/team/tools.git#refs/heads/main//plugins/demo' --target gemini +agentplugins add 'https://git.example.com/team/tools.git#0123456789012345678901234567890123456789//plugins/demo' --target gemini + +# Extensionless Git clone endpoint, without archive auto-detection +agentplugins add 'git+https://git.example.com/team/tools' --target gemini + +# Native SSH credentials and host verification +agentplugins add 'ssh://git@git.example.com:2222/team/tools.git#main//plugins/demo' --target gemini +agentplugins add 'git@git.example.com:team/tools.git#main//plugins/demo' --target gemini + +# Provider browser URLs +agentplugins add 'https://github.com/owner/repo/tree/main/plugins/demo' --target gemini +agentplugins add 'https://gitlab.com/group/subgroup/repo/-/tree/main/plugins/demo' --target gemini +agentplugins add 'https://dev.azure.com/org/project/_git/repo?version=GBmain&path=%2Fplugins%2Fdemo' --target gemini + +# Ready ZIP, TAR, TAR.GZ or TGZ packages; extensionless HTTPS is also detected +agentplugins add 'https://downloads.example.com/plugin.zip' --target gemini +agentplugins add 'https://downloads.example.com/plugin.tar.gz' --target gemini +agentplugins add 'https://downloads.example.com/latest-package' --target gemini + +# A complete local archive is installed without portable dependency intent +agentplugins add './plugin@1.0.zip' --target gemini +``` + +`github:owner/repo@FULL_SHA//path` and historical exact `owner/repo@FULL_SHA` +inputs remain compatible. Bare `owner/repo` continues to select the signed +Directory workflow; use an HTTPS clone URL or explicit `github:` prefix for a +direct Git source. SCP syntax supports an explicit SSH user, including SSH host +aliases, and unambiguous dotted hosts without a user. Absolute SCP paths such as +`git@host:/srv/git/repo.git` retain their native meaning. + +GitHub and GitLab repository URLs normalize to clone endpoints. GitLab tree URLs +with `/-/tree/` also work on self-hosted domains. Azure consumes only its recognized +`path`, `version` and `_a=contents` browser parameters; `GB`, `GT` and `GC` identify +a branch, tag or exact commit. Other remote URL queries, passwords and credential +fragments are refused. SSH usernames remain routing information. Azure's +username-only organization spelling is normalized away. + +A tree URL containing a slash in a branch name is resolved against exact remote +heads/tags. If more than one branch/path split matches, UAP refuses the ambiguity; +use a clone URL with `#refs/heads/branch/name//package/path`. An encoded slash is not +a way to resolve provider URL ambiguity. Ordinary endpoint escaping is preserved. + +## Saved intent, updates and restore + +Eligible remote adds are saved by default. `--alias` chooses the dependency's +logical name; `--no-save` keeps an imperative installation. Existing client +admission rules still apply. Local archive paths remain imperative because they +cannot be restored portably on another computer. + +```toml +schema_version = 1 + +[dependencies.tools] +git = "ssh://git@git.example.com/team/tools.git" +ref = "refs/heads/main" +path = "plugins/demo" + +[dependencies.release] +archive = "https://downloads.example.com/plugin.tar.gz" +path = "plugins/demo" +# Optional: sha256 = "<64 lowercase hex characters>" + +[targets] +gemini = ["tools", "release"] +``` + +The lock records the normalized endpoint and logical package path. Git pins an +exact commit and package tree/manifest digests. Archives additionally pin the +SHA256 of the original downloaded bytes; their immutable revision is +`sha256:`. The archive wrapper directory is not part of the +logical path, so changing a provider's versioned wrapper does not change ownership. + +A moving branch/default Git requirement or archive endpoint can advance through +`update`/`deps update`. Exact commits and explicitly requested archive checksums +stay pinned until the manifest is intentionally edited. Locked sync, repair and +cold restore do not refresh refs or accept a new archive stream in place of the +recorded bytes. If an archive endpoint replaces old bytes and the old package is +not available in the local verified store, locked restore fails before mutation. +Keep versioned stable release URLs for durable archive restoration. + +To pin a CLI archive download and optionally select a nested logical package: + +```bash +agentplugins add 'https://downloads.example.com/plugins.zip#sha256=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef&path=plugins%2Fdemo' --target gemini +``` + +For a moving archive with an explicit nested path, use the `archive`/`path` manifest +table above. Without an explicit path, a root `plugin.json` takes precedence; one +nested package may be selected automatically. Multiple candidates produce a +bounded diagnostic requiring an explicit path. A malformed root manifest does +not fall back to a nested package. Archive selection reuses one retained download. + +Different endpoints remain distinct sources even when their package bytes match. +The shared content store holds bytes; selected lock provenance supplies ownership +and immutable source identity on every load. Historical GitHub and signed +Directory identities remain unchanged. Copy only `uap.toml` and `uap.lock` for a +portable restore, then use `install --locked` or `deps sync --locked --yes`. + +## Credentials and archive policy + +Direct Git uses native Git credential helpers, trusted user Git/SSH configuration, +SSH agent/key access and configured CA certificates. Provision credentials and +SSH known-host trust before invoking UAP. Native transport is bounded and +noninteractive; UAP does not silently accept an unknown SSH host, embed secrets in +portable files, allow a local-file/remote-helper rewrite, or disable TLS +verification. Repository checkout uses isolated Git configuration so package +attributes cannot run user filters/hooks. Signed Directory/Discovery acquisition +retains its separate anonymous immutable GitHub contract. + +HTTPS archives are credential-free downloads from a stable query-free URL. +Bounded HTTPS redirects may use transient signed CDN queries; those URLs and query +tokens are neither persisted nor included in errors. Downgrades, redirect userinfo +and fragments are rejected, and headers/cookies are not carried to another hop. +An extensionless HTTPS input first attempts safe archive recognition; only a +nonarchive response can try native Git. Recognized but corrupt, oversized, +incompatible or unselectable archives never fall back to Git. + +| Boundary | Supported policy | +| --- | --- | +| Formats | Ordinary single-disk ZIP (store/deflate, no ZIP64), TAR, one-member gzip TAR; `.tar.gz` and `.tgz` are equivalent | +| Download | At most 256 MiB retained original bytes; bounded two-minute acquisition | +| Expansion | At most 256 MiB file payload and at most 256 MiB decompressed TAR stream | +| Individual file | At most 64 MiB | +| Entries and paths | At most 10,000; TAR metadata and implicit parent paths are counted | +| Depth | At most 64 portable path segments | +| Members | Regular files/directories only; no links, sparse entries, devices, sockets, FIFOs, encrypted ZIP or multi-volume archives | +| Names | Portable NFC relative names, no traversal/absolute/drive/UNC/backslash/control names or case/Unicode/file-directory collisions; exactly one conventional leading `./` is accepted | +| Framing | Exact ZIP records/end/comment, two TAR zero end blocks with aligned zero padding, one checksum-validated gzip member; no archive preamble, undeclared trailer or concatenated archive | +| Filename | Recognized initial suffix must match detected format; extensionless URLs use bytes; redirect filenames and Content-Type do not override original format policy | + +Acquisition never executes package code. The unchanged standard loader, existing +security assessment and each target client's supported-component checks still run +before installation. A successful transport fixture is not native agent runtime +certification or release qualification. diff --git a/docs/UAP_DEPENDENCIES.md b/docs/UAP_DEPENDENCIES.md index d2b98f3b1..29d51c3fc 100644 --- a/docs/UAP_DEPENDENCIES.md +++ b/docs/UAP_DEPENDENCIES.md @@ -29,6 +29,27 @@ codex = ["review", "docs", "context7"] claude = ["review"] ``` +Git sources may use a canonical GitHub repository, any public/private HTTPS clone +endpoint, an `ssh://` endpoint or native SCP syntax. Direct Git uses existing +native credential helpers/SSH trust and pins an immutable commit. Ready HTTPS +archives use a separate source table: + +```toml +[dependencies.release] +archive = "https://downloads.example.com/plugin.tar.gz" +path = "plugins/demo" +# Optional exact original-byte pin: sha256 = "<64 lowercase hex characters>" +``` + +ZIP, TAR, TAR.GZ/TGZ and extensionless HTTPS downloads use bounded format +recognition and strict extraction. The lock records original-byte SHA256 in +addition to selected package tree/manifest digests; moving endpoints cannot +replace those bytes during locked restore or repair. Local archives are usable +through imperative add, and are not portable dependency requirements. See +[Direct complete package sources](DIRECT_PACKAGE_SOURCES.md) for source grammar, +provider browser URLs, native credentials, selection and exact archive limits. +The standard `plugin.json` format is unchanged; raw skill files are not imported. + Git dependencies use exactly one of `commit` and `ref`; omitting both records the repository default branch intent. Registry tables retain exact directory `release` selection. A string under `[dependencies]` selects a Directory package @@ -55,7 +76,7 @@ The same `[dependencies]` section always describes the manifest owner's direct requirements. Installing a plugin through the managed workflow installs its full dependency closure. Published package shortcuts must use canonical `owner/name` IDs; short Directory names are a convenience for the user's root manifest only. -Git and exact registry-release tables use the same syntax in either context. +Git, archive and exact registry-release tables use the same syntax in either context. An absent package TOML, or one containing only `schema_version = 1`, declares no dependencies. An invalid or unreadable TOML is an error. @@ -101,10 +122,10 @@ agentplugins deps recover ``` `lock` resolves changed selectors and preserves compatible pins. `update -` rechecks a moving Git ref or a registry range, updating the selected +` rechecks a moving Git ref, unpinned archive endpoint or registry range, updating the selected root and related dependency pins while preserving independent pins. Registry ranges stay unchanged in `uap.toml`; apply the new lock with install/sync. Exact -commit, release or semver selectors require an intent change instead. +commit, archive checksum, release or semver selectors require an intent change instead. `tree` reads the saved graph offline, showing roots, versions and consumers; shared nodes and cycles are references. JSON contains sorted `roots` and diff --git a/docs/plans/source-transports-e2e/sources-code-review-r1.md b/docs/plans/source-transports-e2e/sources-code-review-r1.md index e46e17ea0..17f080c8c 100644 --- a/docs/plans/source-transports-e2e/sources-code-review-r1.md +++ b/docs/plans/source-transports-e2e/sources-code-review-r1.md @@ -154,3 +154,26 @@ Focused source model/parser tests and the new native regressions must pass on the corrective commit before R1 GO. Parent owns independent R2 routing, publication and the later cumulative S5 review. Do not interpret this review as an executable release or native agent-client runtime qualification. + +## Corrective implementation evidence (author; reviewer verification pending) + +- `c1f8d232` privately enumerates configured URL-scoped TLS verification key names + in the same Git repository context and overrides each true for the command. + Trusted native includes/auth/SSH/CA semantics remain available. A nonempty + `GIT_SSL_NO_VERIFY=false` was tested and rejected as an unsafe solution for the + actual Git binary; that variable is stripped instead. +- The authoritative `GIT_ALLOW_PROTOCOL=https:ssh` limits actual transport despite + protocol-specific user config or URL rewrites. Only an explicitly injected + `URLForRepo` local fixture mapping can add file transport. +- Actual native regressions reject untrusted TLS under scoped user config, reject + the same setting at an insteadOf-rewritten HTTPS endpoint, reject an HTTPS to + local-file rewrite, and reject HTTPS downgrade redirects. Native HTTPS/private + credential-helper/SSH/host-verification tests passed in4.156s. +- Shared parser/model fixes reject decoded SSH username controls and nonportable + paths on new URL Git intent while preserving historical Repository admission. + `16d58ac7` distinguishes explicit local/Windows archive paths containing `@` + from genuine `owner/repo@release.zip` shorthand. Dotted userless SCP has an + explicit regression. All changed source/model/acquisition lint gates passed. + +This section records the author's evidence; independent reviewer GO is recorded +only after the reviewer performs fix verification. diff --git a/docs/plans/source-transports-e2e/sources-code-review-r2.md b/docs/plans/source-transports-e2e/sources-code-review-r2.md new file mode 100644 index 000000000..ce1f6312b --- /dev/null +++ b/docs/plans/source-transports-e2e/sources-code-review-r2.md @@ -0,0 +1,67 @@ +# Source transports cumulative implementation review R2 + +Independent reviewer: project_core, 2026-10-23. Frozen starting tree `501d7f5610cd584d81a801059a817f7cb5f61657`, cumulative source changes after `1a9b25de`. Read-only source worktree review. Archive implementation was separately reviewed twice, including redirect/dot-prefix amendments and subsequent cancellation/suffix corrections; this review concentrates on native transport corrections and S5 lifecycle/cache integration. + +## Decision + +**Changes required: R2-1 P1 and R2-2 P2 below.** The source lead confirmed both and is implementing narrow fixes and regressions. The architecture, typed source/lock model, native credential boundary and cache-provenance direction are approved. Final GO requires independent fix verification on a frozen follow-up tree and the focused lifecycle tests. + +## R2-1 — P1: imperative generic Git/archive repair uses a source hash as a URL + +`cli/internal/agentpluginscli/lifecycle.go:repairSource` retains its legacy empty-Repository fallback of `CanonicalSource + "#" + revision`. New typed sources deliberately use `git-url-v1:` / `archive-url-v1:` as canonical stable identity; these are not acquisition endpoints. Consequently installations created with new generic Git or archive `--no-save` cannot repair through either the grouped or legacy single-target CLI path. Saved dependency repair takes another typed route, so the initial saved-only S5 journey test does not expose the failure. Explicit local archives have the same reconstruction issue and cannot safely invent a new portable path dependency or drop the original-byte pin. + +Fix both imperative repair routes with recorded typed coordinates: stored endpoint and logical package path plus the selected **binding's exact installed revision**, archive original-byte SHA256, tree digest and manifest digest. Preserve the existing standard loader, security/confirmation gates and source identity checks. Do not resolve today's default ref/download or infer a source from a canonical identity hash. Local archive coordinates remain explicit local source semantics. + +Required regressions: new `--no-save` generic Git, remote archive and explicit local archive installs; advance/change their source, then repair the installed pin (or refuse changed unavailable archive bytes before writes). Also strengthen the saved fixture: its initial repair runs immediately after update while the remote still equals the installed pin; advance the remote again before repair, including cold changed-archive byte-pin refusal with unchanged state/intent/lock. + +## R2-2 — P2: new command-local snapshot caching detaches cancellation + +`deps_resolver.go:dependencyResolver.finishLoaded` newly copies potentially large acquired packages into the command-local snapshot cache using `context.Background()`. Its existing content-store Put also uses Background. This drops the operation's cancellation/deadline during the new resealing/caching boundary. It may finish another bounded but expensive copy after cancellation, contrary to the reviewed command cancellation contract. + +Thread the existing operation context through `finishLoaded` and its production callers, including Put; retain cleanup of both the original loaded package and any failed/cancelled retained snapshot. Add a cancellation regression proving no retained command snapshot or durable store/intent publication after cancellation. No new cancellation mechanism is needed. + +## Provisional observation closed + +The reviewer initially questioned encoded repository separator validation based on an earlier archive worktree's `normalizedURL`. The current source tree preserves RawPath. An independent public-parser probe confirmed both `git+https://github.com/acme%2Fdemo.git` and plain HTTPS forms reject the input. No grammar change is requested for this closed observation. + +## Component assessment + +| Boundary | Assessment | +| --- | --- | +| Pure source grammar and portable model | GO. Explicit Git/archive variants, one-of source counting, archive selector/pin identity, provider browser/ref handling and stable tuple keys are explicit; historical GitHub/Directory key forms remain separate. | +| Native Git policy corrections | GO by code review. Direct native commands retain trusted user authentication and CA/agent settings; command-local policy restores TLS verification including URL-specific overrides and only permits HTTPS/SSH. File transport is confined to the explicit test mapping. Read/checkout phases use isolated Git configuration; fetched repository content cannot activate user filters/hooks. | +| Signed Directory/Discovery | GO. Signed immutable GitHub acquisition remains on the anonymous runner and new direct inputs do not become signed Directory selectors. | +| Shared store and command cache identity | GO direction. Candidate description, materialization and pinned replay rebuild SourceIdentity from the selected pin, never first-writer metadata. Command cache keys include stable source, immutable revision (archive byte checksum), tree and algorithm; snapshot consumers reseal bytes. Cancellation correction required above. | +| Backward GitHub ownership | GO. Legacy source reconstruction preserves historical Repository/source-binding hint semantics; direct transport provenance is replaced by the selected legacy lock before admission. | +| Saved generic source lifecycle | GO direction. Source kind/URL/archive SHA propagate through snapshots, acquired requests, state source binding and inspection; installed repair reconstructs binding revision. Additional old-pin drift fixture required above. | +| Imperative generic source lifecycle | Changes required under R2-1. | +| Confirmation and private preview | GO direction. New saved resolution retains acquired bytes in command-local snapshots, keeps declined/dry-run store/intent unpublished and preserves existing security evaluation. Operation context must remain attached. | +| Project integration seam | GO direction. New helpers operate on their received App; merge must preserve project-selected App, scope/root in every new request and selected environment IDs. Actual generic Git/archive project journey follows qualified adapter integration. | + +## Verification ledger + +- Read native transport policy, pure parser/provider handling, additive source/lock/coordinator variants, lock-derived provenance, all new S5 dispatch/cache/acquired-request helpers and their lifecycle projections. +- Independently inspected the archive suffix fix and ran focused real TLS suffix/redirect tests during archive R1 (0.139s); full acquisition suite passed earlier (2.005s). No redundant large archive expansion run in this cumulative review. +- Independent pure parser probe on the frozen source tree rejects the two encoded-separator inputs above. +- Await frozen fix commit and focused native/direct lifecycle/provenance/cancellation verification before final R2 GO. Agent process behavior in these source transport journeys is fixture-backed and does not certify native agent products. + +## Corrective implementation evidence (author; independent verification pending) + +The follow-up implementation adds a recorded typed-source repair loader shared by +grouped and legacy single-target imperative repair. It uses each selected binding's +immutable revision, tree and manifest, and archive original-byte SHA256, together +with the persisted endpoint/logical path. Local archive transport stays local. +A changed unavailable archive refuses before target/state effects; restoring its +exact original stream permits repair. All finishLoaded production callers now +pass the operation context through retained resealing and store Put. + +The focused completion suite passed in13.847s: actual HTTPS/private Git and +native SSH/SCP CLI saved lifecycle; imperative Git/remote archive/local archive +repair after source advance; saved old-pin drift repair and cold changed-archive +refusal; selected-lock shared-store provenance in both population orders; declined +and dry-run no publication; mid-retention cancellation cleanup; single-download +archive selection, wrapper-independent logical paths, malformed-root precedence, +corrupt archive refusal and no raw SKILL import. Parser/model checks passed too. +Agent process responses remain inert fixtures, and no package code runs. + +Independent reviewer verification and GO follow on this frozen corrective commit. diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_download.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_download.go index a1641d340..1564605de 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_download.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_download.go @@ -235,7 +235,7 @@ func (reader *archiveContextReader) Read(body []byte) (int, error) { func archiveExplicit(endpoint string) bool { u, err := url.Parse(endpoint) - return err == nil && sourcespec.HasArchiveSuffix(u.Path) + return err == nil && (sourcespec.HasArchiveSuffix(u.Path) || sourcespec.IsProviderArchiveURL(u)) } func archiveFilenameFormat(endpoint string) archiveFormat { diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport.go index 6b9ef529a..e717f0139 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport.go @@ -112,7 +112,7 @@ func nativeVerificationOptions(ctx context.Context, environ []string, command Co if key == "" { continue } - if len(key) > 8192 || strings.ContainsAny(key, "\r\n") { + if len(key) > 8192 || strings.ContainsAny(key, "=\r\n") { return nil, errors.New("native Git TLS configuration key is invalid") } options = append(options, "-c", key+"=true") diff --git a/install/integrationctl/agentplugins/sourcespec/parse.go b/install/integrationctl/agentplugins/sourcespec/parse.go index fbfa330a2..c0d70fd6b 100644 --- a/install/integrationctl/agentplugins/sourcespec/parse.go +++ b/install/integrationctl/agentplugins/sourcespec/parse.go @@ -10,6 +10,9 @@ import ( // Parse recognizes direct inputs only. Directory names and local directories // remain the caller's grammar; no filesystem existence probe is performed. func Parse(raw string) (Spec, error) { + if spec, recognized, err := parseLocalArchiveSelector(raw); recognized { + return spec, err + } if explicitArchivePath(raw) { if len(raw) > 8192 || strings.TrimSpace(raw) != raw || strings.ContainsFunc(raw, unicode.IsControl) { return Spec{}, errors.New("invalid local archive path") @@ -65,7 +68,7 @@ func parseHTTPS(raw string, out Spec) (Spec, error) { if err != nil { return Spec{}, errors.New("invalid HTTPS source") } - if strings.HasPrefix(u.Fragment, "sha256=") || HasArchiveSuffix(u.Path) { + if strings.HasPrefix(u.Fragment, "sha256=") || HasArchiveSuffix(u.Path) || IsProviderArchiveURL(u) { a, err := parseArchive(u) out.Kind, out.Archive = KindArchive, &a return out, err @@ -98,7 +101,10 @@ func parseArchive(u *url.URL) (Archive, error) { if err != nil { return Archive{}, err } - a := Archive{URL: endpoint} + return parseArchiveFragment(Archive{URL: endpoint}, fragment) +} + +func parseArchiveFragment(a Archive, fragment string) (Archive, error) { if fragment != "" { values, err := url.ParseQuery(fragment) if err != nil || len(values["sha256"]) != 1 || !ValidSHA256(values.Get("sha256")) { @@ -117,3 +123,15 @@ func parseArchive(u *url.URL) (Archive, error) { } return a, nil } + +func parseLocalArchiveSelector(raw string) (Spec, bool, error) { + base, fragment, selected := strings.Cut(raw, "#") + if !selected || !strings.HasPrefix(fragment, "sha256=") || !HasArchiveSuffix(base) || strings.Contains(base, "://") || looksLikeSCP(base) || strings.HasPrefix(base, "github:") { + return Spec{}, false, nil + } + if len(raw) > 8192 || strings.TrimSpace(raw) != raw || strings.ContainsFunc(raw, unicode.IsControl) { + return Spec{}, true, errors.New("invalid local archive selector") + } + archive, err := parseArchiveFragment(Archive{URL: base}, fragment) + return Spec{Requested: raw, Kind: KindLocalArchive, Archive: &archive}, true, err +} diff --git a/install/integrationctl/agentplugins/sourcespec/provider.go b/install/integrationctl/agentplugins/sourcespec/provider.go index 2f1231a3c..012c8f4b0 100644 --- a/install/integrationctl/agentplugins/sourcespec/provider.go +++ b/install/integrationctl/agentplugins/sourcespec/provider.go @@ -158,3 +158,16 @@ func encodedSeparator(raw string) bool { value := strings.ToLower(raw) return strings.Contains(value, "%2f") || strings.Contains(value, "%5c") } + +// IsProviderArchiveURL recognizes provider download endpoints before browser +// repository parsing. An asset need not advertise its format in its filename. +func IsProviderArchiveURL(u *url.URL) bool { + if strings.Contains(u.Path, "/-/archive/") { + return true + } + if !strings.EqualFold(u.Hostname(), "github.com") { + return false + } + parts := strings.Split(strings.Trim(u.Path, "/"), "/") + return len(parts) >= 6 && parts[2] == "releases" && parts[3] == "download" +} diff --git a/install/integrationctl/agentplugins/sourcespec/source_test.go b/install/integrationctl/agentplugins/sourcespec/source_test.go index 2613e2a0b..a953f981e 100644 --- a/install/integrationctl/agentplugins/sourcespec/source_test.go +++ b/install/integrationctl/agentplugins/sourcespec/source_test.go @@ -29,10 +29,13 @@ func TestParseDirectSources(t *testing.T) { {"https://org@dev.azure.com/org/project/_git/repo?version=GBfeature%2Fa&path=%2Fplugins%2Fdemo&_a=contents", KindGit, &Git{URL: "https://dev.azure.com/org/project/_git/repo", Ref: "refs/heads/feature/a", Path: "plugins/demo"}, nil}, {"https://org.visualstudio.com/project/_git/repo?version=GC" + sha, KindGit, &Git{URL: "https://org.visualstudio.com/project/_git/repo", Commit: sha}, nil}, {"https://dev.azure.com/org/project/_git/repo.git?version=GBmain", KindGit, &Git{URL: "https://dev.azure.com/org/project/_git/repo.git", Ref: "refs/heads/main"}, nil}, + {"https://github.com/owner/repo/releases/download/v1/plugin", KindArchive, nil, &Archive{URL: "https://github.com/owner/repo/releases/download/v1/plugin"}}, + {"https://gitlab.example/g/r/-/archive/main/download", KindArchive, nil, &Archive{URL: "https://gitlab.example/g/r/-/archive/main/download"}}, {"https://github.com/owner/repo/releases/download/v1/plugin.zip", KindArchive, nil, &Archive{URL: "https://github.com/owner/repo/releases/download/v1/plugin.zip"}}, {"https://gitlab.com/g/r/-/archive/main/r.tar.gz", KindArchive, nil, &Archive{URL: "https://gitlab.com/g/r/-/archive/main/r.tar.gz"}}, {"https://downloads.test/package#sha256=" + strings.Repeat("b", 64) + "&path=plugins%2Fdemo", KindArchive, nil, &Archive{URL: "https://downloads.test/package", SHA256: strings.Repeat("b", 64), Path: "plugins/demo"}}, {"https://example.test/package", KindHTTPSAuto, nil, &Archive{URL: "https://example.test/package"}}, + {"./plugins.zip#sha256=" + strings.Repeat("b", 64) + "&path=plugins%2Fdemo", KindLocalArchive, nil, &Archive{URL: "./plugins.zip", SHA256: strings.Repeat("b", 64), Path: "plugins/demo"}}, {"./plugin.tgz", KindLocalArchive, nil, &Archive{URL: "./plugin.tgz"}}, {"owner/repo@release.zip", KindGit, &Git{URL: "https://github.com/owner/repo.git", Repository: "owner/repo", Ref: "release.zip"}, nil}, {`C:\tmp\package.zip`, KindLocalArchive, nil, &Archive{URL: `C:\tmp\package.zip`}}, @@ -63,6 +66,7 @@ func TestUnsafeSourcesFailWithoutEchoingInput(t *testing.T) { "https://github.com/o/r/blob/main/SKILL.md", "https://gitlab.com/g/r/-/blob/main/SKILL.md", "https://dev.azure.com/o/p/_git/r?token=TOKEN", "https://dev.azure.com/o/p/_git/r?version=GBa&version=GBb", "https://example.test/r.git#main\nTOKEN", "https://example.test/../r.git", "ssh://git@example.test/repo#main//CON", + "git+https://github.com/acme%2Fdemo.git", "https://github.com/owner%2Frepo.git", "https://gitlab.com/g/r/-/tree/feature%2Fa/pkg", "https://example.test/r%01.git", } { t.Run(raw, func(t *testing.T) { From 738cd3780660262c6fb4e2599df5d05f4e712bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Fri, 23 Oct 2026 19:43:49 +0300 Subject: [PATCH 08/25] fix(sources): isolate native fixtures and satisfy full CI size gates --- cli/internal/agentpluginscli/deps_commands.go | 24 +++-- .../source_transports_e2e_test.go | 8 +- .../adapters/dependencymanifest/decode.go | 90 +++++++++++-------- .../sourceacquisition/git_transport_test.go | 12 ++- .../adapters/statev2/source_key.go | 19 ++++ .../agentplugins/adapters/statev2/store.go | 10 +-- .../agentplugins/environment/graph.go | 32 ++++--- .../agentplugins/environment/model.go | 30 ++++--- 8 files changed, 144 insertions(+), 81 deletions(-) create mode 100644 install/integrationctl/agentplugins/adapters/statev2/source_key.go diff --git a/cli/internal/agentpluginscli/deps_commands.go b/cli/internal/agentpluginscli/deps_commands.go index e2ca21e09..ffac67f2a 100644 --- a/cli/internal/agentpluginscli/deps_commands.go +++ b/cli/internal/agentpluginscli/deps_commands.go @@ -147,20 +147,28 @@ func runDependencyLock(ctx context.Context, app App, opts *options, updateAlias hadPrevious := previous.LockVersion != 0 changed := !hadPrevious || !equalDependencyLock(previous, locked) if allowWrite && changed && !opts.dryRun { - manifestAfter, readErr := os.ReadFile(manifestPath) - if readErr != nil || string(manifestAfter) != string(manifestBefore) { - return dependencyLockResult{}, errors.New("uap.toml changed during lock resolution; rerun deps lock") - } - if err := ensureDependencyStore(ctx, app, locked); err != nil { - return dependencyLockResult{}, err - } - if err := writeDependencyLock(lockPath, locked); err != nil { + if err := publishResolvedDependencyLock(ctx, app, manifestPath, lockPath, manifestBefore, locked); err != nil { return dependencyLockResult{}, err } + } return dependencyLockResult{Manifest: requirements, Lock: locked, LockPath: lockPath, Changed: changed, DryRun: opts.dryRun}, nil } +func publishResolvedDependencyLock(ctx context.Context, app App, manifestPath, lockPath string, manifestBefore []byte, locked environment.Lock) error { + manifestAfter, readErr := os.ReadFile(manifestPath) + if readErr != nil || string(manifestAfter) != string(manifestBefore) { + return errors.New("uap.toml changed during lock resolution; rerun deps lock") + } + if err := ensureDependencyStore(ctx, app, locked); err != nil { + return err + } + if err := writeDependencyLock(lockPath, locked); err != nil { + return err + } + return nil +} + func derefLock(lock *environment.Lock) environment.Lock { if lock == nil { return environment.Lock{} diff --git a/cli/internal/agentpluginscli/source_transports_e2e_test.go b/cli/internal/agentpluginscli/source_transports_e2e_test.go index c5f68d50f..9253afde4 100644 --- a/cli/internal/agentpluginscli/source_transports_e2e_test.go +++ b/cli/internal/agentpluginscli/source_transports_e2e_test.go @@ -69,9 +69,15 @@ func newDirectTransportFixture(t *testing.T, format string, private bool) *direc home := t.TempDir() t.Setenv("HOME", home) t.Setenv("XDG_CONFIG_HOME", home) + // Empty helper resets inherited osxkeychain/GCM; Schannel otherwise ignores + // an injected CA bundle. These affect disposable fixture HOME only. See + // https://git-scm.com/docs/gitcredentials and https://git-scm.com/docs/git-config. + config := "[credential]\n helper =\n[http]\n schannelUseSSLCAInfo = true\n" if private { - dependencyWrite(t, filepath.Join(home, ".gitconfig"), "[credential \""+fixture.server.URL+"\"]\n helper = \"!f() { printf 'username=fixture\\npassword=disposable-fixture-token\\n'; }; f\"\n") + config += "[credential \"" + fixture.server.URL + "\"]\n helper = \"!f() { printf 'username=fixture\\npassword=disposable-fixture-token\\n'; }; f\"\n" } + dependencyWrite(t, filepath.Join(home, ".gitconfig"), config) + return fixture } diff --git a/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go b/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go index 9e20dbef6..41a63f9b2 100644 --- a/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go +++ b/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go @@ -193,57 +193,73 @@ func decodeRawDependency(value any) (rawDependency, error) { func requirementFromRaw(alias string, raw rawDependency) (environment.Requirement, error) { if raw.Version != "" { - if raw.Git != "" || raw.Archive != "" || raw.SHA256 != "" || raw.Path != "" || raw.Ref != "" || raw.Commit != "" || raw.Registry != "" || raw.Distribution != "" || raw.Release != nil { - return environment.Requirement{}, errors.New("version shortcut cannot contain source table fields") - } - selector, err := rawSelector(raw) - return environment.Requirement{Alias: alias, Registry: &environment.RegistrySource{Distribution: alias, Selector: selector}}, err + return versionRequirement(alias, raw) } selector, err := rawSelector(raw) if err != nil { return environment.Requirement{}, err } - gitSource, registrySource := raw.Git != "", raw.Registry != "" || raw.Distribution != "" || raw.Release != nil - count := 0 - for _, yes := range []bool{gitSource, registrySource, raw.Archive != ""} { - if yes { - count++ - } - } - if count != 1 { + if rawSourceCount(raw) != 1 { return environment.Requirement{}, errors.New("dependency must contain exactly one git, archive or registry source") } if raw.Archive != "" { - if raw.Ref != "" || raw.Commit != "" { - return environment.Requirement{}, errors.New("archive source cannot contain Git selectors") - } - endpoint, err := sourcespec.NormalizeArchiveURL(raw.Archive) - if err != nil { - return environment.Requirement{}, err - } - return environment.Requirement{Alias: alias, Archive: &environment.ArchiveSource{URL: endpoint, Path: raw.Path, SHA256: raw.SHA256}}, nil + return archiveRequirement(alias, raw) } if raw.SHA256 != "" { return environment.Requirement{}, errors.New("sha256 is only valid for archive sources") } - if gitSource { - if raw.Registry != "" || raw.Distribution != "" || raw.Release != nil { - return environment.Requirement{}, errors.New("git source cannot contain registry fields") - } - endpoint := raw.Git - if !strings.Contains(endpoint, ":") { - endpoint = "https://github.com/" + endpoint + ".git" - } - normalized, repository, err := sourcespec.NormalizeGitURL(endpoint) - if err != nil { - return environment.Requirement{}, err - } - source := &environment.GitSource{Repository: repository, Path: raw.Path, Selector: selector} - if repository == "" { - source.URL = normalized + if raw.Git != "" { + return gitRequirement(alias, raw, selector) + } + return registryRequirement(alias, raw, selector) +} + +func rawSourceCount(raw rawDependency) int { + count := 0 + for _, source := range []bool{raw.Git != "", raw.Registry != "" || raw.Distribution != "" || raw.Release != nil, raw.Archive != ""} { + if source { + count++ } - return environment.Requirement{Alias: alias, Git: source}, nil } + return count +} + +func versionRequirement(alias string, raw rawDependency) (environment.Requirement, error) { + if rawSourceCount(raw) != 0 || raw.SHA256 != "" || raw.Path != "" || raw.Ref != "" || raw.Commit != "" { + return environment.Requirement{}, errors.New("version shortcut cannot contain source table fields") + } + selector, err := rawSelector(raw) + return environment.Requirement{Alias: alias, Registry: &environment.RegistrySource{Distribution: alias, Selector: selector}}, err +} + +func archiveRequirement(alias string, raw rawDependency) (environment.Requirement, error) { + if raw.Ref != "" || raw.Commit != "" { + return environment.Requirement{}, errors.New("archive source cannot contain Git selectors") + } + endpoint, err := sourcespec.NormalizeArchiveURL(raw.Archive) + if err != nil { + return environment.Requirement{}, err + } + return environment.Requirement{Alias: alias, Archive: &environment.ArchiveSource{URL: endpoint, Path: raw.Path, SHA256: raw.SHA256}}, nil +} + +func gitRequirement(alias string, raw rawDependency, selector environment.Selector) (environment.Requirement, error) { + endpoint := raw.Git + if !strings.Contains(endpoint, ":") { + endpoint = "https://github.com/" + endpoint + ".git" + } + normalized, repository, err := sourcespec.NormalizeGitURL(endpoint) + if err != nil { + return environment.Requirement{}, err + } + source := &environment.GitSource{Repository: repository, Path: raw.Path, Selector: selector} + if repository == "" { + source.URL = normalized + } + return environment.Requirement{Alias: alias, Git: source}, nil +} + +func registryRequirement(alias string, raw rawDependency, selector environment.Selector) (environment.Requirement, error) { if raw.Ref != "" || raw.Commit != "" { return environment.Requirement{}, errors.New("registry source cannot contain git selectors") } diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport_test.go index 1e259d7f9..e2c45c9d8 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport_test.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_transport_test.go @@ -111,6 +111,7 @@ func TestNativeHTTPSGitPublicAndPrivateCredentialHelper(t *testing.T) { endpoint := server.URL + "/repo.git" source := sourcespec.Git{URL: endpoint} acquirer := Acquirer{TempRoot: t.TempDir()} + configureNativeFixtureCredentials(t, home, server.URL, false) if private { _, err := acquirer.ResolveGit(t.Context(), source, "default", false) if err == nil || strings.Contains(err.Error(), "fixture-token-never-log") { @@ -149,14 +150,16 @@ func TestNativeHTTPSGitPublicAndPrivateCredentialHelper(t *testing.T) { func configureNativeFixtureCredentials(t *testing.T, home, origin string, private bool) { t.Helper() hooks := filepath.Join(home, "hooks") - if err := os.Mkdir(hooks, 0o700); err != nil { + if err := os.MkdirAll(hooks, 0o700); err != nil { t.Fatal(err) } hook := "#!/bin/sh\ntouch '" + filepath.Join(home, "hook-ran") + "'\n" if err := os.WriteFile(filepath.Join(hooks, "reference-transaction"), []byte(hook), 0o700); err != nil { t.Fatal(err) } - config := "[core]\n hooksPath = " + hooks + "\n[filter \"trap\"]\n smudge = touch '" + filepath.Join(home, "filter-ran") + "'; cat\n required = true\n" + // Reset inherited helpers and opt the fixture CA into Windows Schannel; see + // https://git-scm.com/docs/gitcredentials and git-config#httpschannelUseSSLCAInfo. + config := "[credential]\n helper =\n[http]\n schannelUseSSLCAInfo = true\n[core]\n hooksPath = " + hooks + "\n[filter \"trap\"]\n smudge = touch '" + filepath.Join(home, "filter-ran") + "'; cat\n required = true\n" if private { config += "[credential \"" + origin + "\"]\n helper = \"!f() { printf 'username=fixture\\npassword=fixture-token-never-log\\n'; }; f\"\n useHttpPath = true\n" } @@ -170,7 +173,10 @@ func TestNativeGitBrowserSlashRefAndAmbiguity(t *testing.T) { nativeFixtureGit(t, work, "branch", "feature/a") nativeFixtureGit(t, work, "push", "--quiet", filepath.Join(root, "repo.git"), "feature/a") server := nativeHTTPSFixture(t, root, false) - t.Setenv("HOME", t.TempDir()) + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", home) + configureNativeFixtureCredentials(t, home, server.URL, false) acquirer := Acquirer{TempRoot: t.TempDir()} source := sourcespec.Git{URL: server.URL + "/repo.git", Browser: &sourcespec.BrowserTree{Provider: "gitlab", Suffix: "feature/a/pkg"}} resolved, err := acquirer.ResolveGitSource(context.Background(), source) diff --git a/install/integrationctl/agentplugins/adapters/statev2/source_key.go b/install/integrationctl/agentplugins/adapters/statev2/source_key.go new file mode 100644 index 000000000..477637024 --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/statev2/source_key.go @@ -0,0 +1,19 @@ +package statev2 + +import ( + "strings" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +func installedSourceKey(installation domain.Installation) string { + sourceKey := "github:" + strings.ToLower(installation.Source.Repository) + ":" + installation.Source.PackageSubpath + if installation.Source.SourceKind != "" { + sourceKey = sourcespec.Key(sourcespec.Kind(installation.Source.SourceKind), installation.Source.SourceURL, installation.Source.PackageSubpath) + } + if installation.Directory != nil { + sourceKey = "directory:" + strings.ToLower(installation.Directory.DistributionID) + } + return sourceKey +} diff --git a/install/integrationctl/agentplugins/adapters/statev2/store.go b/install/integrationctl/agentplugins/adapters/statev2/store.go index 01850e1bc..96c8bdcb8 100644 --- a/install/integrationctl/agentplugins/adapters/statev2/store.go +++ b/install/integrationctl/agentplugins/adapters/statev2/store.go @@ -14,7 +14,6 @@ import ( "github.com/777genius/plugin-kit-ai/install/integrationctl/adapters/atomicfile" "github.com/777genius/plugin-kit-ai/install/integrationctl/adapters/pathpolicy" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" - "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" ) const ( @@ -221,14 +220,7 @@ func Validate(state domain.StateFileV2) error { if !ok || binding.ClientID != receipt.ClientID || installation.Clients[binding.ClientBindingID].ClientBindingID != binding.ClientBindingID { return fmt.Errorf("%s references unknown client binding", prefix) } - sourceKey := "github:" + strings.ToLower(installation.Source.Repository) + ":" + installation.Source.PackageSubpath - if installation.Source.SourceKind != "" { - sourceKey = sourcespec.Key(sourcespec.Kind(installation.Source.SourceKind), installation.Source.SourceURL, installation.Source.PackageSubpath) - } - if installation.Directory != nil { - sourceKey = "directory:" + strings.ToLower(installation.Directory.DistributionID) - } - if receipt.SourceKey != sourceKey { + if receipt.SourceKey != installedSourceKey(installation) { return fmt.Errorf("%s source key does not match installed identity", prefix) } if receipt.PackageDigest != installation.Source.TreeDigest && receipt.PackageDigest != managedBindingDigest(binding) { diff --git a/install/integrationctl/agentplugins/environment/graph.go b/install/integrationctl/agentplugins/environment/graph.go index c578890ec..ed3aff9ff 100644 --- a/install/integrationctl/agentplugins/environment/graph.go +++ b/install/integrationctl/agentplugins/environment/graph.go @@ -159,20 +159,28 @@ func validateEdges(packages map[string]LockedPackage) error { if err := edge.Selector.Validate(); err != nil { return fmt.Errorf("%s: invalid required_by selector: %w", pkg.Alias, err) } - if pkg.Source.Git != nil { - if edge.Selector != pkg.Source.Git.Requested { - return fmt.Errorf("%s: Git selector differs from consumer %s", pkg.Alias, edge.Alias) - } - } else if pkg.Source.Archive != nil { - if edge.Selector != (Selector{Kind: SelectorArchive, Value: pkg.Source.Archive.RequestedSHA256}) { - return fmt.Errorf("%s: archive selector differs from consumer %s", pkg.Alias, edge.Alias) - } - } else if edge.Selector.Kind != SelectorRelease && edge.Selector.Kind != SelectorVersion && edge.Selector.Kind != SelectorRange { - return fmt.Errorf("%s: invalid registry dependency selector", pkg.Alias) - } else if !registrySelectorMatches(edge.Selector, pkg) { - return fmt.Errorf("%s: version %q does not satisfy consumer %s selector %s", pkg.Alias, pkg.Version, edge.Alias, edge.Selector.Value) + if err := validateRequiredBySource(pkg, edge); err != nil { + return err } + + } + } + return nil +} + +func validateRequiredBySource(pkg LockedPackage, edge RequiredBy) error { + if pkg.Source.Git != nil { + if edge.Selector != pkg.Source.Git.Requested { + return fmt.Errorf("%s: Git selector differs from consumer %s", pkg.Alias, edge.Alias) + } + } else if pkg.Source.Archive != nil { + if edge.Selector != (Selector{Kind: SelectorArchive, Value: pkg.Source.Archive.RequestedSHA256}) { + return fmt.Errorf("%s: archive selector differs from consumer %s", pkg.Alias, edge.Alias) } + } else if edge.Selector.Kind != SelectorRelease && edge.Selector.Kind != SelectorVersion && edge.Selector.Kind != SelectorRange { + return fmt.Errorf("%s: invalid registry dependency selector", pkg.Alias) + } else if !registrySelectorMatches(edge.Selector, pkg) { + return fmt.Errorf("%s: version %q does not satisfy consumer %s selector %s", pkg.Alias, pkg.Version, edge.Alias, edge.Selector.Value) } return nil } diff --git a/install/integrationctl/agentplugins/environment/model.go b/install/integrationctl/agentplugins/environment/model.go index b7d9caafb..75cede35c 100644 --- a/install/integrationctl/agentplugins/environment/model.go +++ b/install/integrationctl/agentplugins/environment/model.go @@ -187,10 +187,7 @@ func (selector Selector) Validate() error { return errors.New("commit selector requires one full lowercase 40-character SHA") } case SelectorRef: - value := strings.TrimSpace(selector.Value) - if value == "" || value != selector.Value || strings.ContainsAny(value, " \t\r\n~^:?*[\\") || strings.HasPrefix(value, "-") || path.Clean(value) != value { - return errors.New("ref selector is not a safe exact Git ref") - } + return validateRefSelector(selector.Value) case SelectorDefault: if selector.Value != "" { return errors.New("default selector must have an empty value") @@ -198,19 +195,30 @@ func (selector Selector) Validate() error { case SelectorVersion, SelectorRange: return validateVersionSelector(selector) case SelectorRelease: - if selector.Value == "" || strings.TrimSpace(selector.Value) != selector.Value { - return errors.New("release selector must be a canonical nonzero sequence") - } - var sequence uint64 - if _, err := fmt.Sscanf(selector.Value, "%d", &sequence); err != nil || sequence == 0 || fmt.Sprint(sequence) != selector.Value { - return errors.New("release selector must be a canonical nonzero sequence") - } + return validateReleaseSelector(selector.Value) default: return fmt.Errorf("unsupported selector kind %q", selector.Kind) } return nil } +func validateRefSelector(value string) error { + if value == "" || strings.TrimSpace(value) != value || strings.ContainsAny(value, " \t\r\n~^:?*[\\") || strings.HasPrefix(value, "-") || path.Clean(value) != value { + return errors.New("ref selector is not a safe exact Git ref") + } + return nil +} +func validateReleaseSelector(value string) error { + if value == "" || strings.TrimSpace(value) != value { + return errors.New("release selector must be a canonical nonzero sequence") + } + var sequence uint64 + if _, err := fmt.Sscanf(value, "%d", &sequence); err != nil || sequence == 0 || fmt.Sprint(sequence) != value { + return errors.New("release selector must be a canonical nonzero sequence") + } + return nil +} + func validateSubpath(value string) error { if value == "" { return nil From cc602063e71e0d62faded266d8682d3968258022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Fri, 23 Oct 2026 19:53:21 +0300 Subject: [PATCH 09/25] test(sources): require native SSH and strengthen provenance contracts --- .github/workflows/source-transports.yml | 68 +++++++++++++++++ .../source_identity_e2e_test.go | 10 ++- .../agentpluginscli/source_ssh_e2e_test.go | 3 + .../source_transports_e2e_test.go | 74 +++++++++++++------ docs/DIRECT_PACKAGE_SOURCES.md | 18 ++++- .../sources-code-review-r1.md | 45 +++++------ .../sources-code-review-r2.md | 35 +++------ .../sourceacquisition/git_ssh_test.go | 3 + 8 files changed, 185 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/source-transports.yml diff --git a/.github/workflows/source-transports.yml b/.github/workflows/source-transports.yml new file mode 100644 index 000000000..656f855e8 --- /dev/null +++ b/.github/workflows/source-transports.yml @@ -0,0 +1,68 @@ +name: Source transports + +on: + pull_request: + paths: + - 'install/integrationctl/agentplugins/**' + - 'cli/internal/agentpluginscli/**' + - 'go.work*' + - '**/go.mod' + - '**/go.sum' + - '.github/workflows/source-transports.yml' + push: + branches: [main, 'feat/source-transports-e2e'] + paths: + - 'install/integrationctl/agentplugins/**' + - 'cli/internal/agentpluginscli/**' + - 'go.work*' + - '**/go.mod' + - '**/go.sum' + - '.github/workflows/source-transports.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: source-transports-${{ github.ref }} + cancel-in-progress: true + +jobs: + native-ssh: + name: native-ssh-linux + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + UAP_REQUIRE_NATIVE_SSH: '1' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: '1.26.10' + cache: false + - name: Require real OpenSSH tooling + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y openssh-server + sudo install -d -m 0755 /run/sshd + for executable in sshd ssh ssh-keygen git jq; do + command -v "$executable" + done + - name: Verify native Git SSH and host authentication + working-directory: install/integrationctl/agentplugins + shell: bash + run: | + set -euo pipefail + go test -count=1 -timeout=3m -json ./adapters/sourceacquisition -run '^TestNativeSSHGitWithIsolatedKeysAndHostVerification$' | tee native-ssh.json + jq -s -e 'any(.[]; .Action == "pass" and .Test == "TestNativeSSHGitWithIsolatedKeysAndHostVerification") and all(.[]; .Action != "skip")' native-ssh.json + - name: Verify saved CLI SSH URL and SCP lifecycle + working-directory: cli + shell: bash + run: | + set -euo pipefail + go test -count=1 -timeout=3m -json ./internal/agentpluginscli -run '^TestDirectSourceSavedLifecycleRealSSH$' | tee cli-ssh.json + jq -s -e 'any(.[]; .Action == "pass" and .Test == "TestDirectSourceSavedLifecycleRealSSH/ssh-url") and any(.[]; .Action == "pass" and .Test == "TestDirectSourceSavedLifecycleRealSSH/scp") and all(.[]; .Action != "skip")' cli-ssh.json diff --git a/cli/internal/agentpluginscli/source_identity_e2e_test.go b/cli/internal/agentpluginscli/source_identity_e2e_test.go index 8b36bcd96..3f057c53c 100644 --- a/cli/internal/agentpluginscli/source_identity_e2e_test.go +++ b/cli/internal/agentpluginscli/source_identity_e2e_test.go @@ -12,23 +12,25 @@ import ( "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" ) -func TestDirectSourcesWithIdenticalBytesKeepSelectedLockProvenance(t *testing.T) { +func TestDirectSourcesWithSameTreeKeepSelectedLockProvenance(t *testing.T) { for _, order := range [][]string{{"a", "b"}, {"b", "a"}} { t.Run(strings.Join(order, "-"), func(t *testing.T) { transport := newDirectTransportFixture(t, "zip", false) sharedStore := filepath.Join(t.TempDir(), "shared-store") coldStore := filepath.Join(t.TempDir(), "shared-cold-store") - var previousDigest, previousKey string + var previousDigest, previousKey, previousSHA string for _, name := range order { + transport.payload = directArchiveBytes(t, transport.work, "zip", "provider-"+name+"/") fixture := directCLIEnvironment(t, transport) fixture.app.EnvironmentStoreRoot = sharedStore endpoint := transport.server.URL + "/" + name + ".zip" dependencyRun(t, fixture, "add", endpoint, "--alias", name, "--target", "gemini", "--format", "json") pin := directLockedPin(t, fixture) - if previousDigest != "" && (previousDigest != pin.TreeDigest || previousKey == pin.SourceKey) { - t.Fatal("same-byte source facts collapsed") + if previousDigest != "" && (previousDigest != pin.TreeDigest || previousKey == pin.SourceKey || previousSHA == pin.Source.Archive.SHA256) { + t.Fatal("byte-distinct, same-tree archive source facts collapsed") } previousDigest, previousKey = pin.TreeDigest, pin.SourceKey + previousSHA = pin.Source.Archive.SHA256 loaded, err := materializeLockedPackage(context.Background(), fixture.app, pin, false, nil) if err != nil { t.Fatal(err) diff --git a/cli/internal/agentpluginscli/source_ssh_e2e_test.go b/cli/internal/agentpluginscli/source_ssh_e2e_test.go index 521623ece..29b99ac9b 100644 --- a/cli/internal/agentpluginscli/source_ssh_e2e_test.go +++ b/cli/internal/agentpluginscli/source_ssh_e2e_test.go @@ -15,6 +15,9 @@ import ( func TestDirectSourceSavedLifecycleRealSSH(t *testing.T) { sshd, err := exec.LookPath("sshd") if err != nil { + if os.Getenv("UAP_REQUIRE_NATIVE_SSH") == "1" { + t.Fatal("required real SSH CLI fixture needs openssh-server") + } t.Skip("real SSH CLI fixture requires openssh-server") } current, err := user.Current() diff --git a/cli/internal/agentpluginscli/source_transports_e2e_test.go b/cli/internal/agentpluginscli/source_transports_e2e_test.go index 9253afde4..770253ecc 100644 --- a/cli/internal/agentpluginscli/source_transports_e2e_test.go +++ b/cli/internal/agentpluginscli/source_transports_e2e_test.go @@ -29,6 +29,9 @@ type directTransportFixture struct { mu sync.Mutex payload []byte downloads int + cloneProbes int + probeSecrets bool + gitAuth int } func newDirectTransportFixture(t *testing.T, format string, private bool) *directTransportFixture { @@ -43,25 +46,7 @@ func newDirectTransportFixture(t *testing.T, format string, private bool) *direc } backend := &cgi.Handler{Path: git, Args: []string{"http-backend"}, Env: []string{"GIT_PROJECT_ROOT=" + project, "GIT_HTTP_EXPORT_ALL=1"}} fixture.payload = directArchiveBytes(t, fixture.work, format, "provider-v1/") - fixture.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/repo.git/") { - if private { - user, password, ok := r.BasicAuth() - if !ok || user != "fixture" || password != "disposable-fixture-token" { - w.Header().Set("WWW-Authenticate", `Basic realm="fixture"`) - w.WriteHeader(http.StatusUnauthorized) - return - } - } - backend.ServeHTTP(w, r) - return - } - fixture.mu.Lock() - defer fixture.mu.Unlock() - fixture.downloads++ - w.Header().Set("Content-Type", "application/octet-stream") - _, _ = w.Write(fixture.payload) - })) + fixture.server = httptest.NewTLSServer(fixture.handler(backend, private)) t.Cleanup(fixture.server.Close) caPath := filepath.Join(t.TempDir(), "ca.pem") dependencyWrite(t, caPath, string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: fixture.server.Certificate().Raw}))) @@ -81,6 +66,43 @@ func newDirectTransportFixture(t *testing.T, format string, private bool) *direc return fixture } +func (fixture *directTransportFixture) handler(backend http.Handler, private bool) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/clone" { + fixture.mu.Lock() + fixture.cloneProbes++ + fixture.probeSecrets = fixture.probeSecrets || r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" + fixture.mu.Unlock() + w.Header().Set("WWW-Authenticate", `Basic realm="fixture"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + if strings.HasPrefix(r.URL.Path, "/clone/") { + r.URL.Path = strings.Replace(r.URL.Path, "/clone/", "/repo.git/", 1) + } + if strings.HasPrefix(r.URL.Path, "/repo.git/") { + if private { + user, password, ok := r.BasicAuth() + if !ok || user != "fixture" || password != "disposable-fixture-token" { + w.Header().Set("WWW-Authenticate", `Basic realm="fixture"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + fixture.mu.Lock() + fixture.gitAuth++ + fixture.mu.Unlock() + } + backend.ServeHTTP(w, r) + return + } + fixture.mu.Lock() + defer fixture.mu.Unlock() + fixture.downloads++ + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(fixture.payload) + }) +} + func directArchiveBytes(t *testing.T, root, format, wrapper string) []byte { t.Helper() var result bytes.Buffer @@ -182,14 +204,14 @@ func directLockedPin(t *testing.T, fixture cliFixture) environment.LockedPackage // responses are inert fixtures; the standard loader, store, ownership and CLI // transactions are production implementations. func TestDirectSourceSavedLifecycleRealHTTPS(t *testing.T) { - for _, format := range []string{"git", "private-git", "zip", "tar", "tar.gz", "tgz", "extensionless"} { + for _, format := range []string{"git", "private-git", "private-extensionless-git", "zip", "tar", "tar.gz", "tgz", "extensionless"} { t.Run(format, func(t *testing.T) { directSavedLifecycleCase(t, format) }) } } func directSavedLifecycleCase(t *testing.T, format string) { t.Helper() - transport := newDirectTransportFixture(t, format, format == "private-git") + transport := newDirectTransportFixture(t, format, strings.HasPrefix(format, "private-")) fixture := directCLIEnvironment(t, transport) endpoint := transport.server.URL + "/package." + format if format == "git" || format == "private-git" { @@ -198,7 +220,17 @@ func directSavedLifecycleCase(t *testing.T, format string) { if format == "extensionless" { endpoint = transport.server.URL + "/download" } + if format == "private-extensionless-git" { + endpoint = transport.server.URL + "/clone" + } directSavedLifecycleEndpoint(t, fixture, transport, format, endpoint) + if format == "private-extensionless-git" { + transport.mu.Lock() + defer transport.mu.Unlock() + if transport.cloneProbes != 1 || transport.probeSecrets || transport.gitAuth == 0 { + t.Fatalf("private extensionless classification/auth boundary: probes=%d secrets=%v native-auth=%d", transport.cloneProbes, transport.probeSecrets, transport.gitAuth) + } + } } func directSavedLifecycleEndpoint(t *testing.T, fixture cliFixture, transport *directTransportFixture, format, endpoint string) { diff --git a/docs/DIRECT_PACKAGE_SOURCES.md b/docs/DIRECT_PACKAGE_SOURCES.md index aab7323f0..afa67c035 100644 --- a/docs/DIRECT_PACKAGE_SOURCES.md +++ b/docs/DIRECT_PACKAGE_SOURCES.md @@ -109,7 +109,8 @@ nested package may be selected automatically. Multiple candidates produce a bounded diagnostic requiring an explicit path. A malformed root manifest does not fall back to a nested package. Archive selection reuses one retained download. -Different endpoints remain distinct sources even when their package bytes match. +Different endpoints remain distinct sources even when their package trees match. +Original archive bytes retain their own checksum when wrappers or framing differ. The shared content store holds bytes; selected lock provenance supplies ownership and immutable source identity on every load. Historical GitHub and signed Directory identities remain unchanged. Copy only `uap.toml` and `uap.lock` for a @@ -151,3 +152,18 @@ Acquisition never executes package code. The unchanged standard loader, existing security assessment and each target client's supported-component checks still run before installation. A successful transport fixture is not native agent runtime certification or release qualification. + + +## Transport verification boundary + +The `Source transports / native-ssh-linux` workflow provisions OpenSSH on +Ubuntu 24.04 and requires the real native Git SSH fixture and saved CLI SSH URL +and SCP lifecycles to execute. Missing SSH tooling, skipped fixtures or missing +expected test pass events fail the job. Disposable keys use exact known-host +verification; the fixture also refuses an untrusted host. + +Real loopback TLS contracts exercise anonymous extensionless archive recognition, +private extensionless Git fallback through native credentials, and archives with +identical package trees but distinct original-byte checksums in both content-store +population orders. Agent responses in these source tests remain inert fixtures; +native agent runtime and project-scope qualification are separate gates. diff --git a/docs/plans/source-transports-e2e/sources-code-review-r1.md b/docs/plans/source-transports-e2e/sources-code-review-r1.md index 17f080c8c..b67090a20 100644 --- a/docs/plans/source-transports-e2e/sources-code-review-r1.md +++ b/docs/plans/source-transports-e2e/sources-code-review-r1.md @@ -1,5 +1,9 @@ # Direct source implementation review R1 +**Current status: R1 GO after corrective verification at `236c3319`**, limited +to the original S1/S2/S4 findings. The original findings below are retained as +the review history; cumulative S5 integration has its own independent review. + Reviewer: `client_plans`, independent of source implementation. Date: 2026-10-23. Reviewed committed S1/S2/S4 changes from `1a9b25de` through `ec7cc19b` in `work/uap-sources`, using `git show` to exclude the concurrently edited CLI @@ -155,25 +159,22 @@ the corrective commit before R1 GO. Parent owns independent R2 routing, publication and the later cumulative S5 review. Do not interpret this review as an executable release or native agent-client runtime qualification. -## Corrective implementation evidence (author; reviewer verification pending) - -- `c1f8d232` privately enumerates configured URL-scoped TLS verification key names - in the same Git repository context and overrides each true for the command. - Trusted native includes/auth/SSH/CA semantics remain available. A nonempty - `GIT_SSL_NO_VERIFY=false` was tested and rejected as an unsafe solution for the - actual Git binary; that variable is stripped instead. -- The authoritative `GIT_ALLOW_PROTOCOL=https:ssh` limits actual transport despite - protocol-specific user config or URL rewrites. Only an explicitly injected - `URLForRepo` local fixture mapping can add file transport. -- Actual native regressions reject untrusted TLS under scoped user config, reject - the same setting at an insteadOf-rewritten HTTPS endpoint, reject an HTTPS to - local-file rewrite, and reject HTTPS downgrade redirects. Native HTTPS/private - credential-helper/SSH/host-verification tests passed in4.156s. -- Shared parser/model fixes reject decoded SSH username controls and nonportable - paths on new URL Git intent while preserving historical Repository admission. - `16d58ac7` distinguishes explicit local/Windows archive paths containing `@` - from genuine `owner/repo@release.zip` shorthand. Dotted userless SCP has an - explicit regression. All changed source/model/acquisition lint gates passed. - -This section records the author's evidence; independent reviewer GO is recorded -only after the reviewer performs fix verification. +## Corrective verification — 2026-10-23 + +Reviewed fixes `c1f8d232` and `16d58ac7` in frozen cumulative source tree +`236c331983d9356345a6654776e1b30a995aabc5`. All five findings are closed: + +| Finding | Verified correction | +| --- | --- | +| R1-1 TLS verification | Native commands enumerate configured `http.*.sslVerify` key names in the same private repository context and override each at its own specificity, plus the generic key. Values are never read or logged. Actual regressions cover both direct and `insteadOf`-rewritten untrusted TLS endpoints with no HTTP request received. `GIT_SSL_NO_VERIFY=false` was correctly **not** adopted: Git treats a nonempty value as a bypass. | +| R1-2 actual protocol restriction | `GIT_ALLOW_PROTOCOL=https:ssh` independently fences user `protocol.file.allow` and URL rewrites. Local file access is confined to the explicit private `URLForRepo` fixture mapping. Actual native tests reject the configured file rewrite and an HTTPS-to-HTTP redirect. | +| R1-3 local archive precedence | Explicit relative/absolute/Windows archive paths are classified before shorthand `@`; the follow-up preserves `owner/repo@release.zip` as Git and native backslash archive paths as local. | +| R1-4 matching path grammar | New URL Git intent/locked variants validate through shared `NormalizeSubpath`; historical Repository coordinates retain their previous admission grammar. | +| R1-5 decoded SSH controls | Decoded user names reject Unicode controls before endpoint normalization. Valid SSH routing remains separate from credentials. | + +Independent exact-toolchain verification on the frozen tree passed: +`sourcespec` full suite 0.003s; `environment` full suite 0.016s; the four actual +native Git TLS/rewrite/protocol/downgrade regressions 0.547s. Tests used +disposable native configuration/repositories and loopback TLS only. No package +code or external service was executed. The existing native credential/config +stability assumption during an individual command remains unchanged. diff --git a/docs/plans/source-transports-e2e/sources-code-review-r2.md b/docs/plans/source-transports-e2e/sources-code-review-r2.md index ce1f6312b..062960664 100644 --- a/docs/plans/source-transports-e2e/sources-code-review-r2.md +++ b/docs/plans/source-transports-e2e/sources-code-review-r2.md @@ -4,7 +4,7 @@ Independent reviewer: project_core, 2026-10-23. Frozen starting tree `501d7f5610 ## Decision -**Changes required: R2-1 P1 and R2-2 P2 below.** The source lead confirmed both and is implementing narrow fixes and regressions. The architecture, typed source/lock model, native credential boundary and cache-provenance direction are approved. Final GO requires independent fix verification on a frozen follow-up tree and the focused lifecycle tests. +**R2 GO at `236c331983d9356345a6654776e1b30a995aabc5`.** Both findings below are closed after independent code and focused execution verification. Their original findings are retained as review history. The architecture, typed source/lock model, native credential boundary and cache-provenance direction are approved. Cohesive source/project execution remains an integration gate after the qualified adapter stack; it is not a blocker in this source-only review. ## R2-1 — P1: imperative generic Git/archive repair uses a source hash as a URL @@ -43,25 +43,14 @@ The reviewer initially questioned encoded repository separator validation based - Read native transport policy, pure parser/provider handling, additive source/lock/coordinator variants, lock-derived provenance, all new S5 dispatch/cache/acquired-request helpers and their lifecycle projections. - Independently inspected the archive suffix fix and ran focused real TLS suffix/redirect tests during archive R1 (0.139s); full acquisition suite passed earlier (2.005s). No redundant large archive expansion run in this cumulative review. - Independent pure parser probe on the frozen source tree rejects the two encoded-separator inputs above. -- Await frozen fix commit and focused native/direct lifecycle/provenance/cancellation verification before final R2 GO. Agent process behavior in these source transport journeys is fixture-backed and does not certify native agent products. - -## Corrective implementation evidence (author; independent verification pending) - -The follow-up implementation adds a recorded typed-source repair loader shared by -grouped and legacy single-target imperative repair. It uses each selected binding's -immutable revision, tree and manifest, and archive original-byte SHA256, together -with the persisted endpoint/logical path. Local archive transport stays local. -A changed unavailable archive refuses before target/state effects; restoring its -exact original stream permits repair. All finishLoaded production callers now -pass the operation context through retained resealing and store Put. - -The focused completion suite passed in13.847s: actual HTTPS/private Git and -native SSH/SCP CLI saved lifecycle; imperative Git/remote archive/local archive -repair after source advance; saved old-pin drift repair and cold changed-archive -refusal; selected-lock shared-store provenance in both population orders; declined -and dry-run no publication; mid-retention cancellation cleanup; single-download -archive selection, wrapper-independent logical paths, malformed-root precedence, -corrupt archive refusal and no raw SKILL import. Parser/model checks passed too. -Agent process responses remain inert fixtures, and no package code runs. - -Independent reviewer verification and GO follow on this frozen corrective commit. +- Agent process behavior in these source transport journeys is fixture-backed and does not certify native agent products. + +## Independent fix verification + +Reviewed corrective `d35d28633bd939eaee85e6dc3d4b42125cbe5f53` and final frozen `236c331983d9356345a6654776e1b30a995aabc5` read-only. The final follow-up preserves those runtime fixes while splitting helpers to satisfy the existing whole-file gates and isolating native Git test credential/CA configuration. + +- **R2-1 closed.** Both grouped and single imperative repair dispatch recorded typed sources through `loadRecordedDirect`. Git acquisition uses the selected binding's exact revision; archive acquisition uses its tagged original-byte SHA256. Recorded logical path, tree and manifest digests are checked. The acquisition endpoint comes from SourceURL, never the canonical hash key. Explicit local archives retain local coordinates. Saved drift repair retains its installed lock pin after the remote advances; unavailable changed archive bytes fail before target/state/intent/lock writes. +- **R2-2 closed.** `finishLoaded` and all production callers, including content-store Put and executable snapshot retention, receive the operation context. Cancellation cleans the original and failed retained snapshot; the regression asserts no command cache or durable content publication. +- Independently ran the frozen tree's `TestDependencyRetentionHonorsOperationCancellation`, `TestDirectImperativeRepairUsesRecordedBytes`, `TestSavedRepairRetainsOldPinAfterSourceAdvance`, `TestDirectSourceSavedLifecycleRealSSH`, `TestDirectSourcesWithIdenticalBytesKeepSelectedLockProvenance`, `TestDirectSavedPreviewDoesNotPublishStoreOrIntent`, `TestLegacyGitHubLockIdentitySurvivesNativeTransport`, `TestDirectArchiveSelectionUsesOneDownloadAndLogicalPath`, and `TestDirectRecognizedInvalidPackagesNeverInstallOrFallback` with `-count=1 -p=2 -timeout=2m`: **PASS, 7.936s**. + +The focused run covers real SSH/SCP Git and TLS archive transport with inert agent fixtures, exact old-pin repairs and changed-byte refusal, same-bytes distinct provenance, private preview, legacy identity, cancellation cleanup, one-download logical selection and no-fallback malformed package handling. No confirmed review finding remains open. diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go index cc809e10f..c785f9e83 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go @@ -19,6 +19,9 @@ import ( func TestNativeSSHGitWithIsolatedKeysAndHostVerification(t *testing.T) { sshd, err := exec.LookPath("sshd") if err != nil { + if os.Getenv("UAP_REQUIRE_NATIVE_SSH") == "1" { + t.Fatal("required native SSH fixture needs openssh-server") + } t.Skip("native SSH fixture requires openssh-server") } root, _, commit := nativeFixtureRepository(t) From 9df203600fa75a6608677ec80e1609c61d311012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Fri, 23 Oct 2026 19:57:40 +0300 Subject: [PATCH 10/25] docs(sources): record final independent lifecycle review --- .github/workflows/source-transports.yml | 2 +- .../sources-code-review-s5-r2.md | 109 ++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 docs/plans/source-transports-e2e/sources-code-review-s5-r2.md diff --git a/.github/workflows/source-transports.yml b/.github/workflows/source-transports.yml index 656f855e8..4e49d2ce7 100644 --- a/.github/workflows/source-transports.yml +++ b/.github/workflows/source-transports.yml @@ -10,7 +10,7 @@ on: - '**/go.sum' - '.github/workflows/source-transports.yml' push: - branches: [main, 'feat/source-transports-e2e'] + branches: [main] paths: - 'install/integrationctl/agentplugins/**' - 'cli/internal/agentpluginscli/**' diff --git a/docs/plans/source-transports-e2e/sources-code-review-s5-r2.md b/docs/plans/source-transports-e2e/sources-code-review-s5-r2.md new file mode 100644 index 000000000..5f388c930 --- /dev/null +++ b/docs/plans/source-transports-e2e/sources-code-review-s5-r2.md @@ -0,0 +1,109 @@ +# Direct source lifecycle implementation review — S5 round 2 + +**Current status: S5 R2 GO at `816e577a` after corrective verification.** +The initial verification requests below are retained as history and all three +are closed by the final ledger. + +Reviewer: `client_plans`, independent of the source implementation. Initial +frozen tree: `236c331983d9356345a6654776e1b30a995aabc5`, after `501d7f56` and +`d35d2863`. Read-only production review. The archive implementation's internals +are excluded from self-review; its two independent reviews and approved +redirect/member-prefix amendments remain authoritative. + +## Decision + +**Initial assessment: implementation direction GO; three narrow verification +gaps required closure before complete S5 qualification.** No new production correctness blocker was +confirmed in this round. Source lead and parent received the concrete fixture +and CI requests below; these are already explicit requirements of the approved +plan, not a new transport matrix or architecture. + +## S5-R2-1 — distinguish archive bytes in the shared-tree provenance proof + +`TestDirectSourcesWithIdenticalBytesKeepSelectedLockProvenance` exercises both +endpoint/cache population orders but serves the identical ZIP byte stream to +both endpoints. It proves endpoint identity reconstruction, without exercising +the separately required byte-distinct archives whose selected package tree is +identical. A first-writer archive checksum is precisely the metadata that must +not be adopted from the shared tree store. + +Strengthen the existing fixture by varying its physical provider wrapper for +each endpoint (`provider-a/`, `provider-b/`) and assert that original-byte SHA256 +differs while tree digest remains equal. The existing materialize/describe/state +and fresh shared-store restore assertions can then validate checksum provenance +in both orders without new duplicated journeys. + +## S5-R2-2 — prove private extensionless HTTPS fallback end to end + +The existing private Git lifecycle uses `/repo.git`, which the parser classifies +directly as Git. The extensionless lifecycle is a valid archive. Neither runs +the required credential-free HTTPS-auto probe -> nonarchive 401 -> native +credential-backed Git fallback boundary. + +Add one real private extensionless clone endpoint to the existing TLS smart-Git +fixture. Its initial root probe returns 401 without receiving Authorization or +cookies; native Git subsequently authenticates using the disposable helper and +loads the standard package. This proves classification and credential isolation +jointly. Inspection of `acquireDirectPackage` found the expected `KindHTTPSAuto` +plus `ErrNotArchive` gate and no fallback after recognized archives; no new +production fallback engine is requested. + +## S5-R2-3 — require the SSH transport proof in one Linux CI lane + +Both real SSH fixtures skip when `sshd` is absent. No source-transport workflow +in the frozen source tree provisions/asserts it, so a green generic test job +can silently omit SSH/SCP qualification. The approved plan requires a bounded +Linux lane that installs or verifies openssh-server and explicitly runs these +fixtures. Parent assigned a separate source-transport workflow to avoid core +workflow edit contention. Platform-optional local skips remain appropriate; +the required lane must fail if its native prerequisite is missing. + +## Component ledger + +| Boundary | Result | +| --- | --- | +| Source dispatch and old capability fakes | GO: old narrow acquirer remains compatible; new direct transport requires an explicit optional capability. Directory and Discovery retain their separate routes. | +| Browser selectors and saved intent | GO: unresolved browser coordinates are resolved before persistence; final Git endpoint/ref/path and immutable commit enter the lock. Archive wrapper paths remain logical. | +| Generic Git/archive saved add/update | GO: typed requirement/lock variants are carried through candidate acquisition, retained-source admission, moving update, exact checksum/commit retention and graph validation. | +| Shared tree cache provenance | Implementation GO: every described/cache/replay/materialization path reconstructs identity from the selected pin; store metadata remains historical. Cache keys bind stable source + immutable revision + algorithm + tree. Proof gap S5-R2-1 remains. | +| Exact installed repair | GO: both single and grouped imperative repair use `loadRecordedDirect`, selected binding revision, source endpoint/path, archive byte pin and tree/manifest digests. Saved repair uses the old installed pin after the remote advances. Changed unavailable archive bytes fail before state/target mutation. | +| Cancellation and consent | GO: `finishLoaded` threads operation context through resealing and store Put; cleanup handles original/failed retained snapshots. Dry-run/decline retain command-local bytes and do not publish store/intent/lock/ownership. | +| Domain/state source facts | GO: common source binding helper carries kind/URL/archive SHA through new installation, update/switch, grouped commit, inspection and state receipt source-key validation. Legacy source-key/identity forms remain separate. | +| Complete-package boundary | GO: standard loader stays authoritative; malformed root manifests, raw skills and recognized corrupt archives do not install or fall through to Git. | +| Transport proof boundaries | Native Git/HTTPS/SSH and archive fixtures exercise actual transports. Agent process behavior in these journeys is inert fixture behavior, not native Gemini/other client runtime certification. S5-R2-2/3 remain. | +| Project/source integration | Pending combined-branch seam proof owned by parent/core. This source branch predates project core and retains existing user-scope Directory request sites; integration must preserve the selected App/context and run a qualified project source journey. This report does not claim that combined seam is already qualified. | + +## Independent verification so far + +Original S1/S2/S4 findings are independently closed in +`sources-code-review-r1.md`: parser full suite 0.003s, environment full suite +0.016s, four actual native Git scoped/rewrite TLS, protocol-file and downgrade +regressions 0.547s on the frozen tree. The source lead and core reviewer also +reported the bounded cumulative native transport/lifecycle suites passing; +final verification in this round will use the frozen follow-up containing the +three narrow proof corrections. No production files were changed by this +reviewer. + +## Corrective verification — all requests closed + +Frozen correction: `816e577a8f73dbc2fc83a0a5ccaeecc5c9d096e4`. + +| Request | Verified correction | +| --- | --- | +| S5-R2-1 | Existing two-order shared-store fixture now uses different provider wrapper names, explicitly asserts distinct archive SHA256 and source key with equal selected tree, and checks pin-derived identity through materialize, describe, persisted state and fresh shared-store restore. | +| S5-R2-2 | Existing real TLS lifecycle adds a private extensionless `/clone` endpoint. The root probe returns 401, records exactly one probe and no Authorization/cookie, then actual native smart-Git requests authenticate with the disposable helper. Add/update/repair/cold restore complete through the production source/lifecycle composition. | +| S5-R2-3 | Dedicated Ubuntu24.04 source-transport workflow provisions and asserts SSH tools, sets `UAP_REQUIRE_NATIVE_SSH=1`, and requires exact JSON pass events for native host verification plus CLI ssh-url and SCP subcases. Missing tools and skips fail the required lane. Local platform-optional behavior remains unchanged. | + +Independent exact Go1.26.10 execution passed the corrective/repair/cancellation +selection in6.889s. A subsequent explicit complete bounded lifecycle selection +with required SSH mode passed in14.500s, covering all real HTTPS Git/private +Git/private extensionless Git/archive forms, both SSH spellings, both shared +store orders with distinct archive bytes, imperative and saved exact old-pin +repair after source advancement, and retained snapshot cancellation. No real +agent/model/MCP runtime or external service was executed. + +**S5 R2 GO.** No production correction or verification request remains open in +this source review. Parent owns remote CI and combined source/project seam +qualification; those remain separate from this source-only verdict. Parent also +requested avoiding duplicate feature push/PR SSH jobs; that workflow capacity +cleanup does not change the reviewed product or transport proof contract. From 07d9298b909072e1be8315da0df1270c03b6e793 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Fri, 23 Oct 2026 20:17:32 +0300 Subject: [PATCH 11/25] test(sources): make Unix SSH server fixture prerequisites explicit --- cli/internal/agentpluginscli/source_ssh_e2e_test.go | 10 ++++++++++ docs/DIRECT_PACKAGE_SOURCES.md | 7 +++++++ .../adapters/sourceacquisition/git_ssh_test.go | 10 ++++++++++ 3 files changed, 27 insertions(+) diff --git a/cli/internal/agentpluginscli/source_ssh_e2e_test.go b/cli/internal/agentpluginscli/source_ssh_e2e_test.go index 29b99ac9b..18f30c17f 100644 --- a/cli/internal/agentpluginscli/source_ssh_e2e_test.go +++ b/cli/internal/agentpluginscli/source_ssh_e2e_test.go @@ -7,12 +7,22 @@ import ( "os/exec" "os/user" "path/filepath" + "runtime" "strings" "testing" "time" ) func TestDirectSourceSavedLifecycleRealSSH(t *testing.T) { + // This provisions a Unix sshd, POSIX forced upload-pack command and Unix + // repository address. A Windows-hosted OpenSSH server is a distinct fixture; + // its account/path/shell rules are not valid remote source grammar input. + if runtime.GOOS == "windows" { + if os.Getenv("UAP_REQUIRE_NATIVE_SSH") == "1" { + t.Fatal("required real SSH CLI fixture needs a Unix OpenSSH server host") + } + t.Skip("real SSH CLI fixture provisions a Unix OpenSSH server; Windows SSH runtime proof is separate") + } sshd, err := exec.LookPath("sshd") if err != nil { if os.Getenv("UAP_REQUIRE_NATIVE_SSH") == "1" { diff --git a/docs/DIRECT_PACKAGE_SOURCES.md b/docs/DIRECT_PACKAGE_SOURCES.md index afa67c035..d9a81bc3a 100644 --- a/docs/DIRECT_PACKAGE_SOURCES.md +++ b/docs/DIRECT_PACKAGE_SOURCES.md @@ -162,6 +162,13 @@ and SCP lifecycles to execute. Missing SSH tooling, skipped fixtures or missing expected test pass events fail the job. Disposable keys use exact known-host verification; the fixture also refuses an untrusted host. +These loopback server fixtures require a Unix host: they provision Unix sshd +accounts, POSIX forced commands and Unix repository paths. They do not qualify +a Windows-hosted OpenSSH server or a Windows SSH client runtime. The required +Linux job fails on unsupported platform/tooling and cannot turn into a skip. +Windows source grammar and real TLS Git/archive lifecycle contracts remain +separate coverage; production SSH source admission is unchanged. + Real loopback TLS contracts exercise anonymous extensionless archive recognition, private extensionless Git fallback through native credentials, and archives with identical package trees but distinct original-byte checksums in both content-store diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go index c785f9e83..3bdf877c6 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_ssh_test.go @@ -8,6 +8,7 @@ import ( "os/exec" "os/user" "path/filepath" + "runtime" "strings" "testing" "time" @@ -17,6 +18,15 @@ import ( ) func TestNativeSSHGitWithIsolatedKeysAndHostVerification(t *testing.T) { + // The disposable server uses POSIX forced-command quoting, Unix account + // identity and Unix absolute repository paths. Do not feed Windows native + // filesystem spellings into a production remote URL to emulate that server. + if runtime.GOOS == "windows" { + if os.Getenv("UAP_REQUIRE_NATIVE_SSH") == "1" { + t.Fatal("required native SSH fixture needs a Unix OpenSSH server host") + } + t.Skip("native SSH fixture provisions a Unix OpenSSH server; Windows SSH runtime proof is separate") + } sshd, err := exec.LookPath("sshd") if err != nil { if os.Getenv("UAP_REQUIRE_NATIVE_SSH") == "1" { From bb6be98663c3e78fdad337a2bfc78864f5a8b16b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Fri, 23 Oct 2026 22:01:52 +0300 Subject: [PATCH 12/25] docs(sources): record independent Unix SSH fixture verification --- .../sources-code-review-r2.md | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/plans/source-transports-e2e/sources-code-review-r2.md b/docs/plans/source-transports-e2e/sources-code-review-r2.md index 062960664..222c3d557 100644 --- a/docs/plans/source-transports-e2e/sources-code-review-r2.md +++ b/docs/plans/source-transports-e2e/sources-code-review-r2.md @@ -54,3 +54,25 @@ Reviewed corrective `d35d28633bd939eaee85e6dc3d4b42125cbe5f53` and final frozen - Independently ran the frozen tree's `TestDependencyRetentionHonorsOperationCancellation`, `TestDirectImperativeRepairUsesRecordedBytes`, `TestSavedRepairRetainsOldPinAfterSourceAdvance`, `TestDirectSourceSavedLifecycleRealSSH`, `TestDirectSourcesWithIdenticalBytesKeepSelectedLockProvenance`, `TestDirectSavedPreviewDoesNotPublishStoreOrIntent`, `TestLegacyGitHubLockIdentitySurvivesNativeTransport`, `TestDirectArchiveSelectionUsesOneDownloadAndLogicalPath`, and `TestDirectRecognizedInvalidPackagesNeverInstallOrFallback` with `-count=1 -p=2 -timeout=2m`: **PASS, 7.936s**. The focused run covers real SSH/SCP Git and TLS archive transport with inert agent fixtures, exact old-pin repairs and changed-byte refusal, same-bytes distinct provenance, private preview, legacy identity, cancellation cleanup, one-download logical selection and no-fallback malformed package handling. No confirmed review finding remains open. + +## Narrow Unix SSH fixture correction verification + +Independently reviewed `37a459db6f7606d3696350fe4480934f2e9cd9f9`: only the two +loopback SSH fixtures and their evidence documentation change. Their sshd +accounts, POSIX forced command and repository URL spellings require a Unix +server host. Windows now explicitly skips this unsupported fixture platform +only when the required-fixture flag is absent. With `UAP_REQUIRE_NATIVE_SSH=1`, +unsupported platform or unavailable SSH tooling remains a hard failure. No +production URL grammar, SSH admission, authentication policy or host trust is +changed. + +The separate Ubuntu24.04 required source workflow still provisions openssh-server, +checks sshd/ssh/ssh-keygen/git/jq, sets the required flag and requires exact pass +events with zero skip events for native acquisition and both SSH URL/SCP CLI +journeys. It cannot satisfy the gate by skipping the fixture. + +Independent frozen Linux execution with `UAP_REQUIRE_NATIVE_SSH=1`, `-count=1`, +`-p=2` and `-timeout=3m`: native SSH acquisition/host verification **PASS1.570s**; +full CLI SSH URL/SCP lifecycle **PASS4.775s** (both subtests pass, no skips). +**GO** for this narrow correction. Windows SSH runtime certification remains +explicitly unclaimed; separate grammar and real TLS lifecycle coverage remains. From 44dafafa9d8a433de2291b259bc93c5f320d0a89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Sat, 24 Oct 2026 11:51:32 +0300 Subject: [PATCH 13/25] docs(sources): preserve scoped recovery and validation plan --- .../review-corrections-restoration-plan.md | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 docs/plans/source-transports-e2e/review-corrections-restoration-plan.md diff --git a/docs/plans/source-transports-e2e/review-corrections-restoration-plan.md b/docs/plans/source-transports-e2e/review-corrections-restoration-plan.md new file mode 100644 index 000000000..3add41f43 --- /dev/null +++ b/docs/plans/source-transports-e2e/review-corrections-restoration-plan.md @@ -0,0 +1,94 @@ +# Source PR473: fresh reconstruction of the five reviewed corrections + +2026-10-24. Unpublished source8d039073 and its local Git objects were erased by +the runtime reset. Prior source98cb9383, original correction plan/amendments, +two plan GOs and two final code GOs are retained conversation history only. +They do not prove reconstructed bytes or authorize inventing old test results. +This restores their bounded contract against publishedbb6be986. Obtain two fresh +independent plan confirmations before code; run fresh relevant tests/lint and +two independent final code reviews before root publishes the exact checkpoint. + +## 1. Decode archive selectors exactly once + +URL parsing already decodes Fragment. Feed u.EscapedFragment() to the existing +url.ParseQuery decoder and clear both Fragment and RawFragment on the cloned +endpoint before normalization. Local selector grammar is already encoded input +and remains unchanged. Test HTTPS/local parity for literal plus (c++), ampersand, +literal percent/%41, slash, Unicode and ordinary selected paths. Preserve malformed +percent, duplicate/unknown keys, invalid SHA and unsafe path refusals. Do not +change source identity, manifests, archive acquisition or auth behavior. + +## 2. Fresh transitive alias completion without invented ownership + +nodeRequirement must preserve precedence: existing pin, explicit root alias, +completed chosen package, Directory distribution and Git repository shorthand. +Only the final fresh/unowned transitive URL-Git/archive acquisition fallback +uses an empty operational Alias so the real acquired manifest supplies its name. +Remove the invented candidate fallback; do not reserve that name or replace an +explicit/pinned/root alias literally named candidate. Public Requirement.Validate +and SourceKey remain strict and still reject an empty alias. + +Because SourceKey validates Alias before deriving the alias-free source tuple, +the narrow internal graph/acquisition key boundaries may clone ONLY an empty +Alias to a temporary valid scalar for invoking that same strict SourceKey. The +private key helper does not relax endpoint/path/selector validation, introduce +a public option, mutate the original request or persist a placeholder in graph, +lock, receipt, saved declaration or owner identity. Archive completion, like +Git, derives manifest.name only for empty/unqualified aliases. Validate true +manifest alias collisions; do not invent a new name to make them pass. + +Meaningful composed tests must use actual Coordinator, production CLI resolver, +loopback TLS Git/ZIP acquisition, strict loader and digest store. Cover two +fresh archives, two URL Git tuples without Repository shorthand and a mixed +pair, distinct acquired names and RequiredBy edges, explicit root candidate, +existing pinned transitive candidate replay and moving explicit-root update. +Actual request counters must establish genuine reacquisition where required. +No client launcher/model executes. Preserve exact prior lock identity. + +## 3. Controlled Git package-path diagnostics + +Classify only locally established package/path/tree facts with a private typed +error: missing selected directory; exact file/symlink/executable rather than +directory; malformed/outside tree; selected submodule. Existing GitHub public +wording remains embedded unchanged. Direct Git reports a controlled static +reason plus its already validated selected package path instead of falsely +calling every failure a credential problem. Unrelated stat failures remain +unclassified, as do arbitrary runner output/error strings. Context cancellation +or deadline wins first. Never print raw Git stdout/stderr, credentials, tree +bytes, signed redirect data or arbitrary error strings through the new path. +Use tests with secret-looking runner failures to prove redaction and cancellation, +alongside actual selected-path cases; no native transport guard is weakened. + +## 4. Actual ZIP CRC corruption proof + +The old corrupt-payload fixture's fixed offset can alter filename/padding or a +deflate control bit while leaving content unchanged. Derive an actual compressed +payload offset through archive/zip DataOffset and flip a checked byte inside that +payload, avoiding its leading deflate control byte. Add a stored-entry regression +that first proves central metadata remains readable and the actual changed data +produces zip.ErrChecksum. Then require production recognized-archive refusal +without a byte pin or Git fallback and an empty acquisition temporary root. +Do not change production archive code, safety assertions, digest policy or caps. + +## 5. Azure documentation table + +Escape only the two pipes in Azure's selector alternatives inside the GFM table. +Keep its three column delimiters and all GB/GT/GC alternatives intact. + +## Verification and publication boundary + +Use fresh workspace Go1.26.10 and caches; coordinate at most the two bounded +GOMAXPROCS2/-p2 compiler slots now authorized by root. Run targeted new parser, +operational-key/public-strictness, controlled-error/CRC and real composed graph +cases, complete sourcespec/environment/sourceacquisition packages as appropriate, +and relevant actual Direct CLI saved lifecycle/repair/provenance regressions. +Run pinned changed-code and full affected size/architecture lint, gofmt and diff +checks. Do not rerun already-passed unrelated native platforms or fake the genuine +hosted guard. Keep original source/legacy/native-SSH/security behavior and tests. + +Root owns all remote writes again after child connector mutations stalled. +Send each clean checkpoint SHA promptly for expected-head/no-force publication +to the existing source feature PR, never main/release or an unrelated branch. +Any materially new design needs its own two-GO amendment. No old execution is +relabeled current, and no source fixture implies actual native agent or release +qualification. From c19a15c40555ce3c1f2b3aaa9247cb8d02fbc9cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Sat, 24 Oct 2026 11:51:50 +0300 Subject: [PATCH 14/25] docs(sources): record two independent recovery plan approvals --- .../restoration-plan-r1.md | 20 +++++++++++++++++++ .../restoration-plan-r2.md | 13 ++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 docs/plans/source-transports-e2e/restoration-plan-r1.md create mode 100644 docs/plans/source-transports-e2e/restoration-plan-r2.md diff --git a/docs/plans/source-transports-e2e/restoration-plan-r1.md b/docs/plans/source-transports-e2e/restoration-plan-r1.md new file mode 100644 index 000000000..3ab6d8b5d --- /dev/null +++ b/docs/plans/source-transports-e2e/restoration-plan-r1.md @@ -0,0 +1,20 @@ +# Fresh source restoration plan review R1 + +Independent reviewer: finish_codex_claude_qoder. **GO** at +637455418f8247e191022d5dccb5ead1a5921075 after complete plan review. + +EscapedFragment through one ParseQuery and clearing both fragment fields keeps +literal selectors intact. Empty aliases occur only for fresh unowned operational +fallback; private cloned scalar key derivation keeps public Validate/SourceKey +and all tuple validation strict. Explicit root and pinned candidate precedence +must never be replaced. Actual Coordinator/production CLI/TLS Git+ZIP request +counters, RequiredBy, pins and collision cases are meaningful. + +Typed locally established Git-path classification, cancellation first, static +reason and validated selected path prevent secret/raw-runner leakage while +retaining GitHub wording. Actual ZIP DataOffset payload corruption plus a +stored-entry zip.ErrChecksum proof closes the old fixture false positive +without changing production archive security. Exactly two Azure table pipes +remain narrow. Fresh affected tests, both lint gates and two final code GOs +are required. No old execution/native proof or guard change. No blocker or +source edit, Go or native execution by review. diff --git a/docs/plans/source-transports-e2e/restoration-plan-r2.md b/docs/plans/source-transports-e2e/restoration-plan-r2.md new file mode 100644 index 000000000..c9b10782c --- /dev/null +++ b/docs/plans/source-transports-e2e/restoration-plan-r2.md @@ -0,0 +1,13 @@ +# Fresh source restoration plan review R2 + +Independent reviewer: finish_grok_integration. **GO** at +637455418f8247e191022d5dccb5ead1a5921075 after complete five-correction review. + +Exact-once escaped-fragment/both-field-clear parity, only fresh unowned empty +alias with private cloned SourceKey scalar and public strictness, controlled +local path facts with cancellation-first redaction, actual payload DataOffset/ +ErrChecksum proof, and the two Azure pipes faithfully restore the prior approved +contract. Meaningful actual Coordinator/CLI TLS Git+ZIP reacquisition, pins, +candidate and collision tests preserve authority. No guard or production archive +relaxation or new native claim. Historical lost source passes are explicitly not +current evidence. No blocker, source edits, Go or native process in this review. From 58cd90891d974731367f97cbad4c6a9a68f11372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Sat, 24 Oct 2026 12:04:09 +0300 Subject: [PATCH 15/25] fix(sources): restore escaped paths transitive identities and archive integrity checks --- cli/internal/agentpluginscli/deps_resolver.go | 9 + .../source_alias_graph_regression_test.go | 179 ++++++++++++++++++ .../restoration-implementation-evidence.md | 59 ++++++ .../source-transports-e2e/sources-plan.md | 2 +- .../adapters/sourceacquisition/acquirer.go | 21 +- .../archive_crc_regression_test.go | 73 +++++++ .../sourceacquisition/archive_hostile_test.go | 10 +- .../adapters/sourceacquisition/git_direct.go | 4 + .../sourceacquisition/git_package_error.go | 47 +++++ .../git_package_error_test.go | 74 ++++++++ .../environment/acquisition_key_test.go | 41 ++++ .../environment/resolve_expand.go | 14 +- .../agentplugins/environment/resolve_graph.go | 2 +- .../archive_fragment_regression_test.go | 30 +++ .../agentplugins/sourcespec/parse.go | 3 +- 15 files changed, 554 insertions(+), 14 deletions(-) create mode 100644 cli/internal/agentpluginscli/source_alias_graph_regression_test.go create mode 100644 docs/plans/source-transports-e2e/restoration-implementation-evidence.md create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/archive_crc_regression_test.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/git_package_error.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/git_package_error_test.go create mode 100644 install/integrationctl/agentplugins/environment/acquisition_key_test.go create mode 100644 install/integrationctl/agentplugins/sourcespec/archive_fragment_regression_test.go diff --git a/cli/internal/agentpluginscli/deps_resolver.go b/cli/internal/agentpluginscli/deps_resolver.go index 1feeb8a50..bd083579e 100644 --- a/cli/internal/agentpluginscli/deps_resolver.go +++ b/cli/internal/agentpluginscli/deps_resolver.go @@ -145,6 +145,9 @@ func (resolver dependencyResolver) resolveArchive(ctx context.Context, requireme _ = loaded.cleanup() return environment.LockedPackage{}, errors.New("archive requirement must explicitly select its logical package path") } + if strings.Contains(locked.Alias, "/") || locked.Alias == "" { + locked.Alias = dependencyManifestAlias(loaded.envelope.Manifest.Name) + } return locked, resolver.finishLoaded(ctx, loaded, locked) } @@ -236,6 +239,12 @@ func githubIdentity(repository, commit, packagePath string) string { } func mustSourceKey(requirement environment.Requirement) string { + // A fresh transitive acquisition has no owner alias until manifest loading. + // This copy only permits strict alias-free source-tuple derivation; the + // temporary scalar is never installed or persisted as binding identity. + if requirement.Alias == "" { + requirement.Alias = "source-acquisition" + } key, _ := requirement.SourceKey() return key } diff --git a/cli/internal/agentpluginscli/source_alias_graph_regression_test.go b/cli/internal/agentpluginscli/source_alias_graph_regression_test.go new file mode 100644 index 000000000..e310a75ae --- /dev/null +++ b/cli/internal/agentpluginscli/source_alias_graph_regression_test.go @@ -0,0 +1,179 @@ +package agentpluginscli + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "sync" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" +) + +type aliasGraphFixture struct { + cli cliFixture + transport *directTransportFixture + root environment.Requirement + mu sync.Mutex + requests map[string]int + archives map[string][]byte + childNames []string +} + +func aliasGraphPackage(t *testing.T, root, name string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(root, "skills", name), 0700); err != nil { + t.Fatal(err) + } + dependencyWrite(t, filepath.Join(root, "plugin.json"), fmt.Sprintf(`{"$schema":%q,"name":%q,"version":"1.0.0"}`, domain.PluginSchemaV1, name)) + dependencyWrite(t, filepath.Join(root, "skills", name, "SKILL.md"), fmt.Sprintf("---\nname: %s\ndescription: Inert transitive fixture\n---\nText only.\n", name)) +} + +func newAliasGraphFixture(t *testing.T, kinds []string, rootAlias string, collision bool) *aliasGraphFixture { + t.Helper() + f := &aliasGraphFixture{transport: newDirectTransportFixture(t, "zip", false), requests: map[string]int{}, archives: map[string][]byte{}, childNames: []string{"first-name", "second-name"}} + if collision { + f.childNames[1] = f.childNames[0] + } + oldHandler := f.transport.server.Config.Handler + f.transport.server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + f.requests[r.URL.Path]++ + payload, archive := f.archives[r.URL.Path] + f.mu.Unlock() + if archive { + _, _ = w.Write(payload) + return + } + oldHandler.ServeHTTP(w, r) + }) + manifest := "schema_version = 1\n" + for i, kind := range kinds { + name := f.childNames[i] + path := fmt.Sprintf("plugins/child-%d", i) + manifest += fmt.Sprintf("[dependencies.local-%d]\n", i) + if kind == "git" { + aliasGraphPackage(t, filepath.Join(f.transport.work, path), name) + manifest += fmt.Sprintf("git = %q\npath = %q\n", f.transport.server.URL+"/repo.git", path) + } else { + root := t.TempDir() + aliasGraphPackage(t, root, name) + endpoint := fmt.Sprintf("/child-%d.zip", i) + f.archives[endpoint] = directArchiveBytes(t, root, "zip", "publisher-wrapper/") + manifest += fmt.Sprintf("archive = %q\n", f.transport.server.URL+endpoint) + } + } + if strings.Contains(strings.Join(kinds, ","), "git") { + dependencyGit(t, f.transport.work, "add", ".") + dependencyGit(t, f.transport.work, "commit", "-m", "inert URL Git graph packages") + dependencyGit(t, f.transport.work, "push", "--quiet", f.transport.remote, "main") + } + root := t.TempDir() + aliasGraphPackage(t, root, "root-manifest") + dependencyWrite(t, filepath.Join(root, "uap.toml"), manifest) + f.archives["/root.zip"] = directArchiveBytes(t, root, "zip", "root-wrapper/") + f.root = environment.Requirement{Alias: rootAlias, Archive: &environment.ArchiveSource{URL: f.transport.server.URL + "/root.zip"}} + f.cli = directCLIEnvironment(t, f.transport) + return f +} + +func (f *aliasGraphFixture) requirements() environment.Requirements { + return environment.Requirements{SchemaVersion: 1, Dependencies: map[string]environment.Requirement{f.root.Alias: f.root}, Targets: map[string][]string{"gemini": {f.root.Alias}}} +} + +func (f *aliasGraphFixture) resolve(t *testing.T, requirements environment.Requirements, previous environment.Lock, update string) environment.Lock { + t.Helper() + lock, err := (environment.Coordinator{Resolver: dependencyResolver{app: f.cli.app, requirements: requirements}}).Resolve(context.Background(), requirements, previous, update) + if err != nil { + t.Fatal(err) + } + return lock +} + +func (f *aliasGraphFixture) requestCount() int { + f.mu.Lock() + defer f.mu.Unlock() + count := 0 + for _, n := range f.requests { + count += n + } + return count +} + +func TestDirectTransitiveArchivesAndURLGitUseAcquiredNames(t *testing.T) { + for _, kinds := range [][]string{{"archive", "archive"}, {"git", "git"}, {"git", "archive"}} { + t.Run(strings.Join(kinds, "+"), func(t *testing.T) { + f := newAliasGraphFixture(t, kinds, "candidate", false) + requirements := f.requirements() + lock := f.resolve(t, requirements, environment.Lock{}, "") + if len(lock.Packages) != 3 { + t.Fatalf("lost distinct source tuples: %+v", lock.Packages) + } + aliases := map[string]environment.LockedPackage{} + for _, pin := range lock.Packages { + aliases[pin.Alias] = pin + if pin.SourceKey == "" || strings.Contains(pin.Alias, "source-acquisition") { + t.Fatal("operational identity entered lock", pin) + } + } + if aliases["candidate"].Source.Archive == nil || aliases["candidate"].Source.Archive.URL != f.root.Archive.URL { + t.Fatal("explicit candidate root alias was replaced") + } + for _, name := range f.childNames { + pin, ok := aliases[name] + if !ok || len(pin.RequiredBy) != 1 || pin.RequiredBy[0].Alias != "candidate" { + t.Fatalf("acquired manifest/edge identity missing: %s %+v", name, pin) + } + } + before := f.requestCount() + replayed := f.resolve(t, requirements, lock, "") + if !reflect.DeepEqual(lock, replayed) || f.requestCount() != before { + t.Fatal("exact replay changed pin or reacquired intact sources") + } + }) + } +} + +func TestDirectTransitivePinnedCandidateSurvivesExplicitUpdate(t *testing.T) { + for _, kind := range []string{"archive", "git"} { + t.Run(kind, func(t *testing.T) { + f := newAliasGraphFixture(t, []string{kind, kind}, "root", false) + requirements := f.requirements() + lock := f.resolve(t, requirements, environment.Lock{}, "") + var pinned environment.LockedPackage + for i := range lock.Packages { + if lock.Packages[i].Alias == f.childNames[0] { + lock.Packages[i].Alias = "candidate" + pinned = lock.Packages[i] + } + } + sort.Slice(lock.Packages, func(i, j int) bool { return lock.Packages[i].Alias < lock.Packages[j].Alias }) + before := f.requestCount() + replayed := f.resolve(t, requirements, lock, "") + if !reflect.DeepEqual(lock, replayed) || f.requestCount() != before { + t.Fatal("pinned literal candidate was replaced or reacquired") + } + requirements.Dependencies["candidate"] = pinned.Requirement() + requirements.Targets["gemini"] = []string{"root", "candidate"} + updated := f.resolve(t, requirements, replayed, "candidate") + if f.requestCount() <= before || !reflect.DeepEqual(updated.Packages, replayed.Packages) { + t.Fatalf("moving explicit candidate did not reacquire exact pin: before%d after%d %+v", before, f.requestCount(), updated.Packages) + } + }) + } +} + +func TestDirectTransitiveRealManifestAliasCollisionRefuses(t *testing.T) { + f := newAliasGraphFixture(t, []string{"archive", "git"}, "root", true) + requirements := f.requirements() + _, err := (environment.Coordinator{Resolver: dependencyResolver{app: f.cli.app, requirements: requirements}}).Resolve(context.Background(), requirements, environment.Lock{}, "") + if err == nil || !strings.Contains(err.Error(), "collides") { + t.Fatalf("real acquired alias collision was renamed or accepted: %v", err) + } +} diff --git a/docs/plans/source-transports-e2e/restoration-implementation-evidence.md b/docs/plans/source-transports-e2e/restoration-implementation-evidence.md new file mode 100644 index 000000000..235a3be2b --- /dev/null +++ b/docs/plans/source-transports-e2e/restoration-implementation-evidence.md @@ -0,0 +1,59 @@ +# Reconstructed source corrections: fresh checkpoint evidence + +Published baselinebb6be986 is preserved. The fresh restoration plan63745541 +received both independent GPT6.1 peer GOs committed7e4fe426 BEFORE source. +Lost source98cb9383/8d039073 passes are historical, not evidence for these bytes. + +## Restored changes + +- HTTPS archive selectors pass EscapedFragment to the existing query decoder; + endpoint copies clear both fragment fields. Literal plus/ampersand/percent, + slash/Unicode and malformed/duplicate/unknown/path refusals have fresh tests. +- Only fresh unowned transitive URL Git/archive fallback uses empty operational + Alias. Existing pin, explicit root, completed selection, Directory and Git + repository precedence remains. Three internal graph/acquisition key uses + clone only an empty Alias to a temporary valid scalar before strict public + SourceKey; public validation is unchanged. Archive acquisition now completes + empty/unqualified alias from actual manifest just as Git does. Nothing reserves + candidate or persists the private scalar as owner identity. +- A private error class marks only own inspected tree/path facts. Its Error + embeds existing GitHub wording. Direct Git exposes a controlled static reason + plus validated selected path, after cancellation-first handling; arbitrary + runner errors, output/tree bytes and unrelated stat failures stay unclassified. + Missing, exact file/executable/symlink root, selected submodule, malformed and + outside tree cases are distinct from credential hints, with secret redaction + and cancellation tests. +- ZIP corruption derives actual compressed-data position from DataOffset and + avoids its first control bit. The independent stored-entry test proves valid + metadata survives while actual changed payload yields zip.ErrChecksum, then + production recognized-archive refusal with no byte pin/Git fallback and empty + temporary acquisition root. Production archive code is unchanged. +- Only the two Azure alternative-selector table pipes are escaped. + +## Fresh actual checks at initial source freeze + +Verified workspace Go1.26.10, workspace caches and GOMAXPROCS2/-p2 were used +within one of the two bounded compiler slots authorized after reset. + +- New targeted parser/key/actual-CRC checks PASS0.006s/0.014s/0.025s. +- New real composed transitive aliases/pins/collision selection PASS5.191s. + Tests use actual Coordinator, production CLI resolver, loopback TLS Git/ZIP, + strict loader and immutable store. Two fresh archives, two URL Git package + tuples and mixed sources yield distinct actual manifest names and RequiredBy + edges while explicit root candidate survives. A renamed existing transitive + candidate pin replays unchanged without requests; selecting it as moving + explicit root performs actual counted HTTP reacquisition and keeps exact pins. + Genuine acquired-name collision refuses. No client or launcher executes. +- Complete sourcespec PASS0.004s; complete environment PASS0.028s; complete + sourceacquisition PASS6.287s, including old and new acquisition protections. +- The initial composed fixture attempt failed before assertions because its + file helper did not create parent directories. The fixture now creates its + own directories; no production condition or authority assertion was relaxed. +- gofmt and authored whitespace checks PASS. + +Existing/new direct saved lifecycle/provenance/repair checks, pinned changed-code +and full-file size/architecture lint, and two fresh independent code reviews are +pending at this checkpoint. No pending gate is counted as passed. No prior native +platform/source result is relabeled, no client/model/auth/trust/scanner operation +ran, and no guard or legacy source was removed or weakened. Root alone publishes +the exact clean checkpoint; merges remain confined to the project feature. diff --git a/docs/plans/source-transports-e2e/sources-plan.md b/docs/plans/source-transports-e2e/sources-plan.md index ba9013d49..a512846c0 100644 --- a/docs/plans/source-transports-e2e/sources-plan.md +++ b/docs/plans/source-transports-e2e/sources-plan.md @@ -64,7 +64,7 @@ Accept these forms: | `[user@]host:team/repo[.git][#REF[//path]]` | SCP-like SSH Git; require a real host and nonempty safe path; accept Azure `git@ssh.dev.azure.com:v3/org/project/repo`; canonicalize spelling deterministically | | `https://github.com/owner/repo[/tree/REF[/path]]` | Strip browser syntax, normalize to repository Git endpoint plus selector/path | | `https://gitlab.com/group[/subgroups]/repo[/-/tree/REF[/path]]` | Support nested groups; `/-/tree/` separates repository from selector/path; same syntax on self-managed hosts containing that explicit marker | -| `https://dev.azure.com/org/[project/]_git/repo[?version=GBbranch|GTtag|GCsha&path=/package]` | Normalize clone endpoint; map GB/GT to explicit refs and GC to commit; decode path once | +| `https://dev.azure.com/org/[project/]_git/repo[?version=GBbranch\|GTtag\|GCsha&path=/package]` | Normalize clone endpoint; map GB/GT to explicit refs and GC to commit; decode path once | | `https://org.visualstudio.com/[project/]_git/repo` | Azure legacy clone/browser host grammar; recognized query fields as above | | HTTPS `.zip`, `.tar`, `.tar.gz`, `.tgz` | Complete archive source | | HTTPS without recognized extension/provider/Git fragment | Bounded HTTPS body acquisition and sniffing; positive ZIP/TAR/gzip-TAR means archive. If the response is not an archive (including 401/404), try native Git for an arbitrary clone endpoint, then report a sanitized combined source error. No credential-bearing HTML/body logging | diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go index 4bc3837ab..3e201486f 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go @@ -308,11 +308,17 @@ func (acquirer Acquirer) acquireGitTransport(ctx context.Context, url, repositor return domain.PackageSnapshot{}, gitAcquisitionFailure(repository, revision, "inspect package tree") } if len(tree) == 0 { - return domain.PackageSnapshot{}, fmt.Errorf("acquire GitHub repository %q at revision %s: inspect package tree: plugin subpath %q does not exist", repository, revision, pathspec) + public := fmt.Errorf("acquire GitHub repository %q at revision %s: inspect package tree: plugin subpath %q does not exist", repository, revision, pathspec) + return domain.PackageSnapshot{}, gitPackageFailure(public, "selected package path does not exist") + } + if reason := selectedGitLeaf(tree, subpath); reason != "" { + return domain.PackageSnapshot{}, gitNonDirectoryFailure(repository, revision, pathspec, reason) } executableFiles, err := executablePathsFromTree(tree, subpath) if err != nil { - return domain.PackageSnapshot{}, fmt.Errorf("acquire GitHub repository %q at revision %s: inspect package tree: %s", repository, revision, publicTreeError(err)) + reason := publicTreeError(err) + public := fmt.Errorf("acquire GitHub repository %q at revision %s: inspect package tree: %s", repository, revision, reason) + return domain.PackageSnapshot{}, gitPackageFailure(public, reason) } if subpath != "" { if _, err := run("-C", repoRoot, "sparse-checkout", "init", "--cone"); err != nil { @@ -330,8 +336,15 @@ func (acquirer Acquirer) acquireGitTransport(ctx context.Context, url, repositor packageRoot = filepath.Join(repoRoot, filepath.FromSlash(subpath)) } info, err := os.Lstat(packageRoot) - if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { - return domain.PackageSnapshot{}, fmt.Errorf("acquire GitHub repository %q at revision %s: plugin subpath %q is not a real directory", repository, revision, pathspec) + if err != nil { + public := fmt.Errorf("acquire GitHub repository %q at revision %s: plugin subpath %q is not a real directory", repository, revision, pathspec) + if os.IsNotExist(err) { + return domain.PackageSnapshot{}, gitPackageFailure(public, "selected package path does not exist") + } + return domain.PackageSnapshot{}, public + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return domain.PackageSnapshot{}, gitNonDirectoryFailure(repository, revision, pathspec, "selected package path is not a real directory") } requestedSource := repository + "@" + revision canonicalSource := "https://github.com/" + requestedSource diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_crc_regression_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_crc_regression_test.go new file mode 100644 index 000000000..f48bc66fa --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_crc_regression_test.go @@ -0,0 +1,73 @@ +package sourceacquisition + +import ( + "archive/zip" + "bytes" + "errors" + "io" + "testing" +) + +func archiveCorruptZIPPayload(t *testing.T, source []byte) []byte { + t.Helper() + reader, err := zip.NewReader(bytes.NewReader(source), int64(len(source))) + if err != nil || len(reader.File) == 0 { + t.Fatalf("invalid ZIP fixture: %v", err) + } + first := reader.File[0] + offset, err := first.DataOffset() + if err != nil || first.CompressedSize64 < 3 || offset < 0 || uint64(offset)+first.CompressedSize64 > uint64(len(source)) { + t.Fatalf("invalid ZIP payload location: offset%d size%d error%v", offset, first.CompressedSize64, err) + } + // This is inside compressed data, away from its leading deflate control bit + // and from names, extra fields, descriptors and central-directory metadata. + index := offset + int64(first.CompressedSize64/2) + corrupt := bytes.Clone(source) + corrupt[index] ^= 1 + return corrupt +} + +func TestArchiveActualStoredZIPChecksumFailureNeverFallsBack(t *testing.T) { + var buffer bytes.Buffer + writer := zip.NewWriter(&buffer) + file, err := writer.CreateHeader(&zip.FileHeader{Name: "plugin.json", Method: zip.Store}) + if err != nil { + t.Fatal(err) + } + const content = `{"$schema":"https://agent-plugins.org/schemas/1.0.0/plugin.schema.json","name":"crc-proof","version":"1.0.0"}` + if _, err := io.WriteString(file, content); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + valid := buffer.Bytes() + reader, err := zip.NewReader(bytes.NewReader(valid), int64(len(valid))) + if err != nil || len(reader.File) != 1 || reader.File[0].Name != "plugin.json" { + t.Fatalf("normal ZIP metadata: %v", err) + } + stream, err := reader.File[0].Open() + if err != nil { + t.Fatal(err) + } + body, readErr := io.ReadAll(stream) + closeErr := stream.Close() + if readErr != nil || closeErr != nil || string(body) != content { + t.Fatalf("normal ZIP data: %q %v %v", body, readErr, closeErr) + } + corrupt := archiveCorruptZIPPayload(t, valid) + reader, err = zip.NewReader(bytes.NewReader(corrupt), int64(len(corrupt))) + if err != nil || len(reader.File) != 1 || reader.File[0].Name != "plugin.json" { + t.Fatalf("payload mutation damaged ZIP metadata: %v", err) + } + stream, err = reader.File[0].Open() + if err != nil { + t.Fatal(err) + } + _, readErr = io.ReadAll(stream) + closeErr = stream.Close() + if !errors.Is(readErr, zip.ErrChecksum) || closeErr != nil { + t.Fatalf("actual stored payload did not fail CRC: %v %v", readErr, closeErr) + } + archiveRejectLocal(t, corrupt) +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_hostile_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_hostile_test.go index 23543e1d7..e4a6c4562 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_hostile_test.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/archive_hostile_test.go @@ -129,10 +129,11 @@ func archiveMutateZIP(t *testing.T, source []byte, mutate func([]byte, []byte, [ func TestArchiveStrictZIPBoundariesAndMetadata(t *testing.T) { valid := archiveFixtureZIP(t, archivePackageEntries(""), "") cases := map[string][]byte{ - "prefix": append([]byte("SFX"), valid...), - "tail": append(bytes.Clone(valid), 0), - "second ZIP": append(bytes.Clone(valid), valid...), - "truncated end": valid[:len(valid)-1], + "prefix": append([]byte("SFX"), valid...), + "tail": append(bytes.Clone(valid), 0), + "second ZIP": append(bytes.Clone(valid), valid...), + "truncated end": valid[:len(valid)-1], + "corrupt payload": archiveCorruptZIPPayload(t, valid), } mutators := map[string]func([]byte, []byte, []byte){ "multiple disks": func(_, _, end []byte) { end[4] = 1 }, @@ -141,7 +142,6 @@ func TestArchiveStrictZIPBoundariesAndMetadata(t *testing.T) { "unsupported method": func(local, central, _ []byte) { local[8] = 99; central[10] = 99 }, "declared oversized": func(_, central, _ []byte) { binary.LittleEndian.PutUint32(central[24:28], 64<<20+1) }, "actual size differs": func(_, central, _ []byte) { binary.LittleEndian.PutUint32(central[24:28], 1) }, - "corrupt payload": func(local, _, _ []byte) { local[40] ^= 1 }, } for name, mutate := range mutators { cases[name] = archiveMutateZIP(t, valid, mutate) diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go index 1db3299dd..90ee10ea1 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go @@ -52,6 +52,10 @@ func (acquirer Acquirer) AcquireGit(ctx context.Context, source sourcespec.Git, if ctx.Err() != nil { return domain.PackageSnapshot{}, ctx.Err() } + var selected *gitPackagePathError + if errors.As(err, &selected) { + return domain.PackageSnapshot{}, fmt.Errorf("acquire direct Git source: package path %q: %s", packagePath, selected.reason) + } return domain.PackageSnapshot{}, fmt.Errorf("acquire direct Git source: immutable fetch or package snapshot failed; verify native Git credentials and repository access") } if expectedTreeDigest != "" && snapshot.TreeDigest != expectedTreeDigest { diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_package_error.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_package_error.go new file mode 100644 index 000000000..0a5fd6f6b --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_package_error.go @@ -0,0 +1,47 @@ +package sourceacquisition + +import ( + "bytes" + "fmt" +) + +// gitPackagePathError is private evidence from our own inspected tree/path. +// Error preserves the GitHub public wording; direct transport callers may +// expose only its controlled reason and their already validated selected path. +type gitPackagePathError struct { + public error + reason string +} + +func (err *gitPackagePathError) Error() string { return err.public.Error() } + +func gitPackageFailure(public error, reason string) error { + return &gitPackagePathError{public: public, reason: reason} +} + +func selectedGitLeaf(tree []byte, subpath string) string { + if subpath == "" { + return "" + } + for _, record := range bytes.Split(tree, []byte{0}) { + metadata, name, ok := bytes.Cut(record, []byte{'\t'}) + fields := bytes.Fields(metadata) + if !ok || len(fields) != 3 || string(name) != subpath { + continue + } + switch string(fields[0]) { + case "100644": + return "selected package path is a file, not a directory" + case "100755": + return "selected package path is an executable file, not a directory" + case "120000": + return "selected package path is a symbolic link, not a real directory" + } + } + return "" +} + +func gitNonDirectoryFailure(repository, revision, selected, reason string) error { + public := fmt.Errorf("acquire GitHub repository %q at revision %s: plugin subpath %q is not a real directory", repository, revision, selected) + return gitPackageFailure(public, reason) +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_package_error_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_package_error_test.go new file mode 100644 index 000000000..088484dce --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_package_error_test.go @@ -0,0 +1,74 @@ +package sourceacquisition + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" +) + +type packageFailureRunner struct { + tree []byte + failure error + cancel context.CancelFunc +} + +func (runner packageFailureRunner) Run(_ context.Context, command Command) ([]byte, error) { + if len(command.Args) > 2 { + switch command.Args[2] { + case "fetch": + return nil, runner.failure + case "rev-parse": + return []byte(strings.Repeat("1", 40) + "\n"), nil + case "ls-tree": + if runner.cancel != nil { + runner.cancel() + } + return runner.tree, nil + } + } + return nil, nil +} + +func TestDirectGitControlledPackageDiagnosticsAndRedaction(t *testing.T) { + const secret = "credential-token-DO-NOT-PRINT" + source := sourcespec.Git{URL: "https://git.test/team/repo.git", Path: "plugin"} + for _, tc := range []struct{ name, tree, want string }{ + {"missing", "", "does not exist"}, + {"file", "100644 blob abc\tplugin\x00", "a file"}, + {"executable", "100755 blob abc\tplugin\x00", "executable file"}, + {"symlink", "120000 blob abc\tplugin\x00", "symbolic link"}, + {"submodule", "160000 commit abc\tplugin/vendor\x00", "submodule"}, + {"outside", "100755 blob abc\toutside/" + secret + "\x00", "outside the plugin subpath"}, + {"malformed", secret + "\x00", "invalid package tree"}, + } { + t.Run(tc.name, func(t *testing.T) { + temporary := t.TempDir() + _, err := (Acquirer{TempRoot: temporary, Runner: packageFailureRunner{tree: []byte(tc.tree)}}).AcquireGit(context.Background(), source, strings.Repeat("1", 40), "") + if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), `package path "plugin"`) || strings.Contains(err.Error(), secret) || strings.Contains(err.Error(), "credentials") { + t.Fatalf("controlled diagnostic missing or leaked: %v", err) + } + archiveAssertEmpty(t, temporary) + }) + } + _, err := (Acquirer{TempRoot: t.TempDir(), Runner: packageFailureRunner{failure: errors.New(secret)}}).AcquireGit(context.Background(), source, strings.Repeat("1", 40), "") + if err == nil || strings.Contains(err.Error(), secret) || !strings.Contains(err.Error(), "credentials") { + t.Fatalf("transport error leaked or was path-classified: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, err = (Acquirer{TempRoot: t.TempDir(), Runner: packageFailureRunner{tree: []byte(secret), cancel: cancel}}).AcquireGit(ctx, source, strings.Repeat("1", 40), "") + if !errors.Is(err, context.Canceled) || strings.Contains(err.Error(), secret) { + t.Fatalf("cancellation lost to diagnostic: %v", err) + } +} + +func TestGitHubPackageDiagnosticKeepsPublicWording(t *testing.T) { + _, err := (Acquirer{TempRoot: t.TempDir(), Runner: packageFailureRunner{}}).AcquireGitHub(context.Background(), "example/plugin", strings.Repeat("1", 40), "plugin") + want := `acquire GitHub repository "example/plugin" at revision ` + strings.Repeat("1", 40) + `: inspect package tree: plugin subpath "plugin" does not exist` + if err == nil || err.Error() != want { + t.Fatalf("GitHub public wording changed: %v", err) + } +} diff --git a/install/integrationctl/agentplugins/environment/acquisition_key_test.go b/install/integrationctl/agentplugins/environment/acquisition_key_test.go new file mode 100644 index 000000000..0f26340fc --- /dev/null +++ b/install/integrationctl/agentplugins/environment/acquisition_key_test.go @@ -0,0 +1,41 @@ +package environment + +import ( + "strings" + "testing" +) + +func TestAcquisitionKeyCopiesOnlyEmptyOperationalAlias(t *testing.T) { + for _, req := range []Requirement{ + {Archive: &ArchiveSource{URL: "https://packages.test/archive.zip"}}, + {Git: &GitSource{URL: "https://git.test/team/repo.git", Selector: Selector{Kind: SelectorDefault}}}, + } { + if req.Validate() == nil { + t.Fatal("public validation admitted empty alias") + } + if _, err := req.SourceKey(); err == nil { + t.Fatal("public SourceKey admitted empty alias") + } + explicit := req + explicit.Alias = "candidate" + want, err := explicit.SourceKey() + if err != nil { + t.Fatal(err) + } + got, err := acquisitionSourceKey(req) + if err != nil || got != want || req.Alias != "" { + t.Fatalf("operational key changed tuple/request: %s %s %+v %v", got, want, req, err) + } + if got, err := acquisitionSourceKey(explicit); err != nil || got != want || explicit.Alias != "candidate" { + t.Fatal("explicit candidate alias changed", err) + } + req.Alias = "bad alias" + if _, err := acquisitionSourceKey(req); err == nil { + t.Fatal("nonempty invalid alias was normalized") + } + } + invalid := Requirement{Archive: &ArchiveSource{URL: "http://unsafe.test/archive.zip", SHA256: strings.Repeat("a", 64)}} + if _, err := acquisitionSourceKey(invalid); err == nil { + t.Fatal("private key bypassed endpoint validation") + } +} diff --git a/install/integrationctl/agentplugins/environment/resolve_expand.go b/install/integrationctl/agentplugins/environment/resolve_expand.go index 476aa9ebf..00434ba11 100644 --- a/install/integrationctl/agentplugins/environment/resolve_expand.go +++ b/install/integrationctl/agentplugins/environment/resolve_expand.go @@ -39,7 +39,7 @@ func (solver *graphSolver) expand(choices map[string]ResolvedPackage) (map[strin } func mergeGraphDemand(nodes map[string]graphNode, parent graphNode, req Requirement) bool { - key, _ := req.SourceKey() + key, _ := acquisitionSourceKey(req) child, exists := nodes[key] changed := !exists if !exists { @@ -98,12 +98,22 @@ func (solver *graphSolver) nodeRequirement(key string, req Requirement, choices } else if req.Git != nil && req.Git.Repository != "" { req.Alias = req.Git.Repository } else { - req.Alias = "candidate" + req.Alias = "" } } return req } +// Acquisition may know a source before its manifest supplies the binding name. +// Only this operational key uses a disposable scalar; public validation and +// every endpoint/path/selector check still run through the strict SourceKey. +func acquisitionSourceKey(req Requirement) (string, error) { + if req.Alias == "" { + req.Alias = "source-acquisition" + } + return req.SourceKey() +} + func rootDeclaration(requirements Requirements, source string) string { for declaration, req := range requirements.Dependencies { key, _ := req.SourceKey() diff --git a/install/integrationctl/agentplugins/environment/resolve_graph.go b/install/integrationctl/agentplugins/environment/resolve_graph.go index 459d81e26..3941e41ff 100644 --- a/install/integrationctl/agentplugins/environment/resolve_graph.go +++ b/install/integrationctl/agentplugins/environment/resolve_graph.go @@ -265,7 +265,7 @@ func (solver *graphSolver) try(ctx context.Context, key string, selected Resolve } func nodeAccepts(node graphNode, pkg LockedPackage) bool { - key, _ := node.requirement.SourceKey() + key, _ := acquisitionSourceKey(node.requirement) if pkg.SourceKey != key { return false } diff --git a/install/integrationctl/agentplugins/sourcespec/archive_fragment_regression_test.go b/install/integrationctl/agentplugins/sourcespec/archive_fragment_regression_test.go new file mode 100644 index 000000000..dde554f75 --- /dev/null +++ b/install/integrationctl/agentplugins/sourcespec/archive_fragment_regression_test.go @@ -0,0 +1,30 @@ +package sourcespec + +import ( + "net/url" + "strings" + "testing" +) + +func TestArchiveHTTPSAndLocalSelectorsDecodeExactlyOnce(t *testing.T) { + checksum := strings.Repeat("a", 64) + for _, selected := range []string{"plugins/c++", "plugins/a&b", "plugins/%41", "plugins/a/b", "plugins/日本語", "ordinary"} { + t.Run(selected, func(t *testing.T) { + selector := "#sha256=" + checksum + "&path=" + url.QueryEscape(selected) + for _, endpoint := range []string{"https://packages.test/plugin.zip", "/tmp/plugin.zip"} { + spec, err := Parse(endpoint + selector) + if err != nil || spec.Archive == nil || spec.Archive.Path != selected || spec.Archive.SHA256 != checksum || strings.Contains(spec.Archive.URL, "#") { + t.Fatalf("selector %s: %+v %v", endpoint, spec, err) + } + } + }) + } + for _, fragment := range []string{ + "sha256=" + checksum + "&path=a%ZZ", "sha256=" + checksum + "&path=a&path=b", + "sha256=" + checksum + "&unknown=x", "sha256=short&path=a", "sha256=" + checksum + "&path=../outside", + } { + if _, err := Parse("https://packages.test/plugin.zip#" + fragment); err == nil { + t.Fatalf("invalid archive fragment admitted: %s", fragment) + } + } +} diff --git a/install/integrationctl/agentplugins/sourcespec/parse.go b/install/integrationctl/agentplugins/sourcespec/parse.go index c0d70fd6b..f9e27a640 100644 --- a/install/integrationctl/agentplugins/sourcespec/parse.go +++ b/install/integrationctl/agentplugins/sourcespec/parse.go @@ -94,9 +94,10 @@ func parseHTTPS(raw string, out Spec) (Spec, error) { } func parseArchive(u *url.URL) (Archive, error) { - fragment := u.Fragment + fragment := u.EscapedFragment() base := *u base.Fragment = "" + base.RawFragment = "" endpoint, err := NormalizeArchiveURL(base.String()) if err != nil { return Archive{}, err From 9c6025fd1b7c6e4682ea29716ca32f6728f065c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Sat, 24 Oct 2026 12:12:26 +0300 Subject: [PATCH 16/25] docs(sources): preserve fresh restoration lifecycle and lint results --- .../restoration-implementation-evidence.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/plans/source-transports-e2e/restoration-implementation-evidence.md b/docs/plans/source-transports-e2e/restoration-implementation-evidence.md index 235a3be2b..4adfbe6d7 100644 --- a/docs/plans/source-transports-e2e/restoration-implementation-evidence.md +++ b/docs/plans/source-transports-e2e/restoration-implementation-evidence.md @@ -51,9 +51,21 @@ within one of the two bounded compiler slots authorized after reset. own directories; no production condition or authority assertion was relaxed. - gofmt and authored whitespace checks PASS. -Existing/new direct saved lifecycle/provenance/repair checks, pinned changed-code -and full-file size/architecture lint, and two fresh independent code reviews are -pending at this checkpoint. No pending gate is counted as passed. No prior native +## Fresh final authored gates + +- Existing/new direct saved lifecycle/provenance/repair/replay selection PASS + 21.212s with actual loopback TLS Git/ZIP. A previous tool response was lost at + context transition; that unobserved result is not counted, and this is the + fresh completed rerun. +- Pinned changed-code lint from pre-code two-GO checkpoint7e4fe426 PASS0issues + across complete sourcespec, environment, sourceacquisition and public CLI. +- Full-file revive/funlen/gocyclo/gocognit/dupl/depguard over those same packages + PASS0issues. No guard or baseline changes were made. + +Two fresh independent code review closures remain pending at this evidence +checkpoint. These leaf results precede a newly required forward integration of +upstream main008370dd and do not certify unconstructed merge bytes. No pending +gate is counted as passed. No prior native platform/source result is relabeled, no client/model/auth/trust/scanner operation ran, and no guard or legacy source was removed or weakened. Root alone publishes the exact clean checkpoint; merges remain confined to the project feature. From 9dded00997fc4c1f4690255321c010fd6dfb6cbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Sat, 24 Oct 2026 12:14:23 +0300 Subject: [PATCH 17/25] docs(sources): plan reviewed integration with upstream standalone skill support --- .../upstream-skills-integration-plan.md | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 docs/plans/source-transports-e2e/upstream-skills-integration-plan.md diff --git a/docs/plans/source-transports-e2e/upstream-skills-integration-plan.md b/docs/plans/source-transports-e2e/upstream-skills-integration-plan.md new file mode 100644 index 000000000..e3119de27 --- /dev/null +++ b/docs/plans/source-transports-e2e/upstream-skills-integration-plan.md @@ -0,0 +1,126 @@ +# Forward integration of accepted standalone Skills and source transports + +2026-10-24. Root observed that source PR473 has no runnable merge CI because +main advanced to008370dd6767fecd19ebf25902f099b102226ba5. Its accepted changes +include standalone versioned GitHub Agent Skills (e3d84d05/#478), desktop and +terminal polish (#488), and landing changes. Preserve them all. This is a +forward merge into the source feature, never a main/release mutation or a +rewrite of accepted upstream PRs. Obtain two independent fresh plan GOs BEFORE +semantic conflict resolutions, and two fresh implementation reviews plus +meaningful current-byte checks after the integration. + +## Frozen inputs and actual conflict inventory + +The reconstructed source implementation is b171c142647297d532764fd120508a07cec79979; +its actual fresh test/lint evidence is preserved by88220d575b5987760be03ee818edec14ada8616e. +Those leaf results do not certify the upcoming merge. Read-only merge-tree +against exact upstream008370dd produced conflict-bearing treeb36dc1c5 and16 +conflicted files: deps_resolver, deps_staging, deps_sync, manifestedit/edit, +dependencymanifest/decode, sourceacquisition/acquirer, statev2/store, +domain/state+types, environment/graph+model, installer/inspect+package, +usecase/binding+group_commit+installation. Automatic merges also touch identity, +environment session and README; review their interactions rather than assuming +textual success proves the contract. New upstream Skills adapter/discovery, +normalizer, picker UI/bridge/API/schema, qualification, release, scanner and +landing files stay accepted upstream bytes unless a necessary interaction is +identified, planned and reviewed. No blanket ours/theirs conflict choice. + +## Combined identity, grammar and lock contract + +Retain both field families: transport SourceKind/SourceURL/ArchiveSHA256, URL Git +and archive sources, plus upstream StableSourceKey/SkillSourceFacts and locked +Skill kind/name/requested path/resolved path/raw digest/normalizer. Clone Skill +facts where upstream clones revision metadata; do not lose facts in installation, +grouped commit, binding or persisted state. Retain transport projection helpers +and extend them to carry the upstream fields once rather than duplicating source +construction. Existing GitHub source keys and historical locks remain unchanged. + +GitHub plugins, direct URL Git plugins, archives, Directory packages and normalized +GitHub Skills remain distinct source-key namespaces. A GitHub Skill retains the +accepted github-skill key including expected skill name and requested path; +requested identity is independent from relocated resolved path. URL Git and +archive keys remain endpoint+package tuple identities, never GitHub coercions. +StableSourceKey is used only for the upstream normalized Skill authority case; +it must not become a caller-controlled override of arbitrary transport identity. + +Manifest decoding/editing retains kind+skill AND archive+sha256 syntax and strict +unknown/duplicate fields, exact-one-source and selector validation. Existing +Skills are GitHub-only: a direct URL Git/archive/registry source may not acquire +kind=skill semantics through the combined grammar. Preserve both strict public +Requirement validation and fresh unowned transitive empty-alias behavior; the +private key scalar remains ephemeral and never changes explicit candidate or +an existing pin. Locked selector validation checks all relevant source-family, +URL/repository, kind/name, path, selector, commit/archive and normalization facts. +Requirement reconstruction preserves both URL and Skill fields exactly. + +## Resolution, materialization, reuse and cold repair + +Share immutable selector resolution without discarding the feature's native +transport branch. Exact GitHub Skill requests route to upstream resolveSkill +before ordinary plugin loading, preserving relocation lookup and raw acquisition, +deterministic normalization and source facts. Ordinary GitHub/URL Git requests +retain actual source transport and verified pin semantics; archive behavior stays +unchanged. Do not normalize a plugin or send a raw Skill through plugin loading. + +The snapshot cache key retains full canonical source key+immutable revision+ +algorithm+artifact digest and, for Skills, expected name/requested path/raw digest/ +normalizer. Distinct raw inputs or normalization contracts cannot share a cached +snapshot merely because normalized artifact bytes happen to match. Pinned Skills +use acquireLockedSkill (raw digest verification then normalized tree verification), +while ordinary source pins use existing acquirePinnedSource. Source projection +from a lock uses skillSnapshotSource only for exact Skill locks and the existing +identityFromLocked path for ordinary GitHub/URL/archive/Directory locks; retain +upstream malformed-lock errors before effects. + +Upstream moved build/applyDependencyRepairs from deps_sync to deps_repair. Keep +one implementation in that upstream location and combine per-target recorded +revision reconstruction: transport URL/archive pin metadata AND upstream Skill +facts/raw digest/normalizer. Missing per-client Skill provenance still refuses +cold repair. Directory publication identity continues to take its existing path. +Repair never resolves a moving ref or substitutes latest bytes for an old pin. +Preserve scoped transaction, scanner/authorization, state ownership and alias +checks; no fallback through legacy engines or trust/auth shortcuts. + +## Git extraction policy and retained guards + +Upstream now distinguishes plugin acquisition's allowed unrelated root gitlinks +from strict raw-Skill tree completeness through allowRootGitlinks and its public +AcquireGitHubSkill APIs. Keep that policy separate from the source feature's +native transport boolean. Use a named private common extraction path or separate +explicit flags so a native URL transport never accidentally grants raw Skill +gitlink leniency. Raw Skills refuse source submodule/incomplete trees; plugin +policy remains the upstream policy. Retain native Git environment isolation, +immutable revision checking, sparse selection, file/path/tree validation, +executability and digest guards, controlled own-path diagnostics, cancellation +priority and runner secret redaction. No native client executes in these checks. + +## Meaningful current-byte validation and review + +Before the merge commit, inspect every resolved conflict and the automatic +identity/environment-session interactions. Verify accepted UI/landing/Skills +consumer files remain unchanged where no interaction requires edits. Preserve +all old tests; change no assertions merely to remove an integration failure. + +Run full environment, sourcespec, manifest decoder/editor, sourceacquisition, +skillsource, state/domain/usecase/installer affected suites and public CLI saved +source plus standalone-Skill flows. Existing upstream Skills tests must continue +to exercise deterministic normalization, raw tree and normalized artifact digest, +relocation and per-target cold repair. Existing/new source tests retain real TLS +Git/ZIP, saved add/sync/update/pin repair/replay, same endpoint different package +tuples, explicit candidate/pin precedence, controlled redaction and actual CRC. + +Add narrow combined regressions for grammar/source-family refusal and round-trip +of both field families; exact source namespace distinction; cache separation on +Skill normalization facts; Skill + direct transport source projection and repair +of recorded per-target old revisions without moving-ref substitution. Use actual +composition and inert/loopback fixtures rather than tautological helper copies. +Run full relevant pinned changed-code and unchanged full-file size/architecture +lint; no architecture snapshot, platform list, baseline or security guard edits. +Run existing upstream picker qualification checks as required by its workflow; +if infrastructure blocks a gate, record the actual blocker rather than a pass. + +Freeze small clean checkpoints with exact input SHAs and current evidence for +root publication. Record independent review findings and fixes, then obtain two +fresh final code GOs. Root alone publishes the forward-merge feature and uses +normal GitHub CI; no source leaf pass, archived evidence or hosted client pass +is relabeled as proof for changed integration bytes or all-client completion. From 4185343e7199c5ed21e53a314b86fe8a91293ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Sat, 24 Oct 2026 12:15:16 +0300 Subject: [PATCH 18/25] docs(sources): preserve two final independent restoration code approvals --- .../restoration-code-reviews.md | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 docs/plans/source-transports-e2e/restoration-code-reviews.md diff --git a/docs/plans/source-transports-e2e/restoration-code-reviews.md b/docs/plans/source-transports-e2e/restoration-code-reviews.md new file mode 100644 index 000000000..fb68333f9 --- /dev/null +++ b/docs/plans/source-transports-e2e/restoration-code-reviews.md @@ -0,0 +1,34 @@ +# Fresh independent reconstructed-source implementation reviews + +Both reviewers are independent GPT6.1 peers. Reviewed final leaf evidence +88220d575b5987760be03ee818edec14ada8616e; production is unchanged b171c142647297d532764fd120508a07cec79979. +These GOs do not certify the upcoming forward integration of main008370dd. + +## R1 — finish_codex_claude_qoder — final GO + +Full15-file delta inspected: private operational key/public strictness and +precedence, actual acquired archive manifest alias, own-tree typed errors/static +reasons/cancellation-first handling/raw-secret redaction/GitHub wording, +EscapedFragment and both cleared fragment fields, actual DataOffset plus stored +ErrChecksum and cleanup proof, and real Coordinator/production CLI TLS Git/ZIP +names/RequiredBy/request counts/pinned candidate/update/collision cases. +Fresh complete packages0.004/0.028/6.287, composed graph5.191, final direct +saved/provenance/repair21.212 and both pinned lint gates0issues close the execution +hold. The final production-to-evidence delta is documentation only and whitespace +is clean. No native/guard/legacy capability lost; no further leaf blocker. +Reviewer launched no duplicate Go or native process. The forthcoming upstream +merge requires its own two-GO plan, current merged tests and reviews. + +## R2 — finish_grok_integration — final leaf GO + +Full production delta and composed graph/CRC/diagnostic regressions inspected. +Empty alias is limited to fresh unowned fallback; the private scalar is cloned +only for strict key calculation, preserving explicit candidate, pins and public +validation. Typed own-path reasons and cancellation-first handling redact runner +bytes. Stored CRC proof establishes intact metadata and ErrChecksum before +production refusal. No production archive or guard widening. Exact final +documentation delta records actual direct lifecycle/provenance/repair21.212 and +pinned changed-code/full-file size+architecture0issues, closing the original +execution hold alongside complete packages and meaningful graph/CRC checks. +No unresolved blocker and no duplicate reviewer compiler/native operation. +This is explicitly a leaf GO, not qualification of upcoming integration bytes. From 9a65dd969ce554aee2a23ee8cb483fc7b9c4e632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D0=B8=D1=8F?= Date: Sat, 24 Oct 2026 12:16:42 +0300 Subject: [PATCH 19/25] docs(sources): preserve two independent upstream integration plan approvals --- ...pstream-skills-integration-plan-reviews.md | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 docs/plans/source-transports-e2e/upstream-skills-integration-plan-reviews.md diff --git a/docs/plans/source-transports-e2e/upstream-skills-integration-plan-reviews.md b/docs/plans/source-transports-e2e/upstream-skills-integration-plan-reviews.md new file mode 100644 index 000000000..4612c44bd --- /dev/null +++ b/docs/plans/source-transports-e2e/upstream-skills-integration-plan-reviews.md @@ -0,0 +1,33 @@ +# Independent forward integration plan reviews + +Both independent GPT6.1 peer reviews cover frozen docs-only plan +f93af5b23cf86f0eba5f7822006ca83ad7bf9bd2 before semantic merge resolutions. + +## R1 — finish_codex_claude_qoder — GO + +Full plan read. Exact008370dd accepted UI/landing/Skill changes preserved; +16 named conflicts plus automatic identity/session interactions explicitly +reviewed without blanket ours/theirs. Both transport and Skill provenance field +families, namespace grammar and locked-selector reconstruction retained. +StableSourceKey is confined to exact validated normalized GitHub Skill authority, +never an arbitrary caller override. Cache key includes raw/name/requested path/ +normalizer/digest, acquireLockedSkill verifies raw then normalized artifact and +per-target cold repair in moved deps_repair preserves the old pin. Distinct +allowRootGitlinks and native URL transport flags, GitHub-only Skills and original +native Git security/cancellation/redaction remain. Combined composition/cache/ +grammar/repair regressions, both existing upstream/source suites, lint and two +fresh merged code GOs are mandatory. Leaf882 GO is not merge proof. No blocker; +reviewer ran no compiler or native process. + +## R2 — finish_grok_integration — GO + +Full16-conflict inventory and automatic interaction contract read. Combined +source families and strict GitHub-only Skill grammar, stable key constrained to +normalized Skill authority, raw/normalized facts through cache/lock/state/repair, +distinct nativeTransport versus allowRootGitlinks, cancellation/secrets and +existing guards are sound. Meaningful current suites, combined namespace/cache/ +repair tests, accepted upstream byte comparisons and two fresh code GOs required. +No blanket resolution, UI rewrite or main mutation. Inherited Skills project +frontend admission is owned by a separate reviewed amendment; accepted upstream +user picker files stay unchanged in this integration. No blocker and no compiler +or native process launched by review. From 571af5a19291adaa8819f021e0e888c622d7645c Mon Sep 17 00:00:00 2001 From: iliya Date: Sat, 24 Oct 2026 14:59:28 +0300 Subject: [PATCH 20/25] fix(skills): preserve portable executable metadata during normalization --- .../skills_full_flow_e2e_test.go | 9 +++- .../adapters/skillsource/normalize.go | 13 +++++- .../adapters/skillsource/skillsource_test.go | 46 +++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/cli/internal/agentpluginscli/skills_full_flow_e2e_test.go b/cli/internal/agentpluginscli/skills_full_flow_e2e_test.go index 1f1f3fc76..5a9f0ac3d 100644 --- a/cli/internal/agentpluginscli/skills_full_flow_e2e_test.go +++ b/cli/internal/agentpluginscli/skills_full_flow_e2e_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "os" "path/filepath" + "runtime" + "slices" "strings" "testing" "time" @@ -40,6 +42,8 @@ func skillsFullFlowFixture(t *testing.T) (savedJourney, string, []byte) { } savedJourneyGit(t, repo, "init", "-b", "main") savedJourneyGit(t, repo, "add", ".") + // Immutable Git executable metadata is independent of host filesystem bits. + savedJourneyGit(t, repo, "update-index", "--chmod=+x", "engineering/review/scripts/check.sh", "docs/review/scripts/check.sh") savedJourneyGit(t, repo, "commit", "-m", "test: disposable raw Skills end-to-end fixture") journey := newSavedJourney(t, repo) journey.cli.app.SourceAcquirer = skillsource.Acquirer{Acquirer: journey.source.Acquirer} @@ -65,8 +69,11 @@ func TestSkillsFullCLIEndToEnd(t *testing.T) { if err != nil || string(got) != string(raw) { t.Fatalf("raw Skill bytes changed: %q %v", got, err) } + if !slices.Contains(pin.ExecutableFiles, "skills/review/scripts/check.sh") || !slices.Contains(snapshot.Metadata.ExecutableFiles, "skills/review/scripts/check.sh") { + t.Fatalf("resource executable metadata missing: lock=%v store=%v", pin.ExecutableFiles, snapshot.Metadata.ExecutableFiles) + } script, err := os.Stat(filepath.Join(snapshot.Root, "skills", "review", "scripts", "check.sh")) - if err != nil || script.Mode()&0100 == 0 { + if err != nil || (runtime.GOOS != "windows" && script.Mode()&0100 == 0) { t.Fatalf("resource executable mode missing: %v %v", script, err) } state := journey.state(t) diff --git a/install/integrationctl/agentplugins/adapters/skillsource/normalize.go b/install/integrationctl/agentplugins/adapters/skillsource/normalize.go index 2269aa2da..927e824ba 100644 --- a/install/integrationctl/agentplugins/adapters/skillsource/normalize.go +++ b/install/integrationctl/agentplugins/adapters/skillsource/normalize.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "os" + "path" "path/filepath" "strings" @@ -61,7 +62,7 @@ func Normalize(ctx context.Context, raw domain.PackageSnapshot, expectedName, st if err = writeSyntheticManifest(temp, stableSourceKey); err != nil { return domain.PackageSnapshot{}, err } - artifact, err := builder.Snapshot(ctx, temp, raw.Source) + artifact, err := snapshotNormalizedTree(ctx, builder, temp, verified, skill.Name) if err != nil { return domain.PackageSnapshot{}, err } @@ -138,3 +139,13 @@ func writeSyntheticManifest(root, stableSourceKey string) error { } return nil } + +// snapshotNormalizedTree seals the portable package after raw content is copied. +// Executable identity belongs to the immutable Git inventory, not host mode bits. +func snapshotNormalizedTree(ctx context.Context, builder packagedigest.Builder, root string, raw domain.PackageSnapshot, skillName string) (domain.PackageSnapshot, error) { + executables := make([]string, 0, len(raw.ExecutableFiles)) + for _, relative := range raw.ExecutableFiles { + executables = append(executables, path.Join("skills", skillName, relative)) + } + return builder.SnapshotWithExecutables(ctx, root, raw.Source, executables) +} diff --git a/install/integrationctl/agentplugins/adapters/skillsource/skillsource_test.go b/install/integrationctl/agentplugins/adapters/skillsource/skillsource_test.go index 9b936139e..75f8035f5 100644 --- a/install/integrationctl/agentplugins/adapters/skillsource/skillsource_test.go +++ b/install/integrationctl/agentplugins/adapters/skillsource/skillsource_test.go @@ -386,3 +386,49 @@ func TestRootRawSkillRejectsGitlinksBeforeSnapshot(t *testing.T) { t.Fatal(err) } } + +// Windows lacks POSIX executable bits. Exercise the real final-sealing boundary +// with non-executable host files so portable Git metadata remains authoritative. +func TestNormalizedTreeSealingUsesPortableExecutableInventory(t *testing.T) { + for _, withExecutable := range []bool{false, true} { + t.Run(fmt.Sprint(withExecutable), func(t *testing.T) { + root := t.TempDir() + put(t, root, "skills/review/SKILL.md", document("review"), 0600) + raw := domain.PackageSnapshot{Source: domain.SourceIdentity{Repository: "example/fixture"}} + expected := []string{} + if withExecutable { + put(t, root, "skills/review/scripts/run", "#!/bin/sh\nprintf resource-only\n", 0600) + raw.ExecutableFiles = []string{"scripts/run"} + expected = []string{"skills/review/scripts/run"} + info, err := os.Stat(filepath.Join(root, "skills/review/scripts/run")) + if err != nil || info.Mode()&0111 != 0 { + t.Fatalf("fixture must have no host executable bits: %v %v", info, err) + } + } + if err := writeSyntheticManifest(root, "github-skill:example/fixture:review:review"); err != nil { + t.Fatal(err) + } + builder := packagedigest.Builder{TempRoot: t.TempDir()} + got, err := snapshotNormalizedTree(context.Background(), builder, root, raw, "review") + if err != nil { + t.Fatal(err) + } + defer func() { _ = packagedigest.Remove(got) }() + if !slices.Equal(got.ExecutableFiles, expected) { + t.Fatalf("portable executable metadata lost: got %v want %v", got.ExecutableFiles, expected) + } + reference, err := builder.SnapshotWithExecutables(context.Background(), root, raw.Source, expected) + if err != nil { + t.Fatal(err) + } + defer func() { _ = packagedigest.Remove(reference) }() + if got.DigestAlgorithm != domain.TreeDigestAlgorithm || got.TreeDigest != reference.TreeDigest { + t.Fatalf("portable tree identity differs: got %s want %s", got.TreeDigest, reference.TreeDigest) + } + body, err := os.ReadFile(filepath.Join(got.Root, "skills/review/SKILL.md")) + if err != nil || string(body) != document("review") { + t.Fatal("sealing changed raw document", err) + } + }) + } +} From 9d1a7033799c8ee793e4103626b3d853e25eade1 Mon Sep 17 00:00:00 2001 From: iliya Date: Sat, 24 Oct 2026 15:54:53 +0300 Subject: [PATCH 21/25] feat(skills): install and restore pinned HTTPS publisher sources --- README.md | 4 +- cli/internal/agentpluginscli/deps_repair.go | 9 +- .../agentpluginscli/deps_repair_source.go | 31 +++ cli/internal/agentpluginscli/deps_resolver.go | 10 +- cli/internal/agentpluginscli/deps_source.go | 6 + cli/internal/agentpluginscli/deps_staging.go | 30 +-- cli/internal/agentpluginscli/deps_sync.go | 3 + .../agentpluginscli/deps_web_skills.go | 77 ++++++ .../agentpluginscli/environment_session.go | 4 +- .../agentpluginscli/manifestedit/edit.go | 4 +- .../agentpluginscli/manifestedit/edit_test.go | 21 ++ cli/internal/agentpluginscli/skills.go | 7 +- cli/internal/agentpluginscli/skills_batch.go | 1 + .../agentpluginscli/skills_batch_selection.go | 21 +- .../agentpluginscli/skills_website.go | 47 ++++ .../skills_website_e2e_test.go | 167 ++++++++++++ docs/AGENT_SKILLS_INSTALLER.md | 48 +++- .../website-skills-evidence.md | 76 ++++++ .../source-transports-e2e/website-skills.md | 75 ++++++ .../adapters/dependencymanifest/decode.go | 35 ++- .../adapters/dependencymanifest/web_test.go | 45 ++++ .../adapters/sourceacquisition/acquirer.go | 48 ---- .../sourceacquisition/acquirer_helpers.go | 58 +++++ .../adapters/sourceacquisition/skill_web.go | 195 ++++++++++++++ .../sourceacquisition/skill_web_index.go | 46 ++++ .../sourceacquisition/skill_web_paths.go | 46 ++++ .../sourceacquisition/skill_web_paths_test.go | 65 +++++ .../sourceacquisition/skill_web_test.go | 244 ++++++++++++++++++ .../adapters/statev2/source_key.go | 2 +- .../adapters/statev2/web_source_key_test.go | 15 ++ .../agentplugins/domain/identity.go | 5 + .../agentplugins/domain/state.go | 17 +- .../agentplugins/domain/types.go | 23 +- .../agentplugins/domain/web_skill.go | 21 ++ .../agentplugins/domain/web_skill_policy.go | 155 +++++++++++ .../domain/web_skill_policy_test.go | 37 +++ .../agentplugins/environment/coordinator.go | 4 + .../agentplugins/environment/graph.go | 18 +- .../agentplugins/environment/model.go | 29 ++- .../agentplugins/environment/resolve_graph.go | 2 +- .../agentplugins/environment/skills.go | 5 +- .../agentplugins/environment/sources.go | 8 +- .../agentplugins/environment/web.go | 26 ++ .../agentplugins/environment/web_test.go | 94 +++++++ .../agentplugins/installer/acquired_skill.go | 26 +- .../agentplugins/installer/acquired_source.go | 6 + .../installer/acquired_web_skill_test.go | 55 ++++ .../agentplugins/installer/prepare.go | 3 + .../agentplugins/usecase/installation.go | 3 +- .../agentplugins/usecase/service.go | 6 +- .../usecase/web_skill_revision_test.go | 24 ++ 51 files changed, 1873 insertions(+), 134 deletions(-) create mode 100644 cli/internal/agentpluginscli/deps_repair_source.go create mode 100644 cli/internal/agentpluginscli/deps_web_skills.go create mode 100644 cli/internal/agentpluginscli/skills_website.go create mode 100644 cli/internal/agentpluginscli/skills_website_e2e_test.go create mode 100644 docs/plans/source-transports-e2e/website-skills-evidence.md create mode 100644 docs/plans/source-transports-e2e/website-skills.md create mode 100644 install/integrationctl/agentplugins/adapters/dependencymanifest/web_test.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer_helpers.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web_index.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web_paths.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web_paths_test.go create mode 100644 install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web_test.go create mode 100644 install/integrationctl/agentplugins/adapters/statev2/web_source_key_test.go create mode 100644 install/integrationctl/agentplugins/domain/web_skill.go create mode 100644 install/integrationctl/agentplugins/domain/web_skill_policy.go create mode 100644 install/integrationctl/agentplugins/domain/web_skill_policy_test.go create mode 100644 install/integrationctl/agentplugins/environment/web.go create mode 100644 install/integrationctl/agentplugins/environment/web_test.go create mode 100644 install/integrationctl/agentplugins/installer/acquired_web_skill_test.go create mode 100644 install/integrationctl/agentplugins/usecase/web_skill_revision_test.go diff --git a/README.md b/README.md index fb9fef578..3e0b77284 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ or backup. Transferring the files does not guarantee offline installation. See [Restore on another machine](docs/UAP_DEPENDENCIES.md#restore-on-another-machine) for confirmation prompts, supported restore behavior and recovery. -Source-build development: [install raw Agent Skills from GitHub](docs/AGENT_SKILLS_INSTALLER.md), +Source-build development: [install raw Agent Skills from GitHub or compatible HTTPS publishers](docs/AGENT_SKILLS_INSTALLER.md), including the local browser picker and versioned saved dependencies. This does not claim availability in an already published installer release. @@ -199,7 +199,7 @@ ZIP/TAR/TAR.GZ/TGZ packages. See [Direct complete package sources](docs/DIRECT_P for syntax, native credentials, immutable archive byte pins and supported limits. These complete-package `agentplugins add` transports do not import standalone skill files as plugins. The separate source-build `agentplugins skills add` -workflow above installs raw GitHub Agent Skills through deterministic normalization. +workflow above installs raw Agent Skills through deterministic normalization. Other manifest formats are ignored and cannot override `plugin.json`. diff --git a/cli/internal/agentpluginscli/deps_repair.go b/cli/internal/agentpluginscli/deps_repair.go index e25168a04..4ccfb8325 100644 --- a/cli/internal/agentpluginscli/deps_repair.go +++ b/cli/internal/agentpluginscli/deps_repair.go @@ -31,13 +31,8 @@ func buildDependencyRepairs(ctx context.Context, app App, state domain.StateFile return nil, errors.New("recorded package revision is missing") } pkg := environment.LockedPackage{Alias: authority.Alias, SourceKey: authority.SourceKey, DigestAlgorithm: domain.TreeDigestAlgorithm, TreeDigest: rev.TreeDigest, ManifestDigest: rev.ManifestDigest, Targets: []string{string(target)}} - pkg.Source.Git = &environment.LockedGitSource{Repository: installation.Source.Repository, URL: installation.Source.SourceURL, Path: installation.Source.PackageSubpath, Commit: rev.ResolvedRevision, Requested: environment.Selector{Kind: environment.SelectorCommit, Value: rev.ResolvedRevision}} - if installation.Source.StableSourceKey != "" { - facts := rev.Skill - if facts == nil { - return nil, errors.New("recorded Skill source provenance unavailable; restore the matching state/lock before repair") - } - pkg.Source.Git = &environment.LockedGitSource{Kind: environment.SkillKind, Skill: facts.Name, Repository: installation.Source.Repository, Path: facts.RequestedPath, Commit: rev.ResolvedRevision, Requested: environment.Selector{Kind: environment.SelectorCommit, Value: rev.ResolvedRevision}, ResolvedSkillPath: facts.ResolvedPath, RawTreeDigest: facts.RawTreeDigest, Normalizer: facts.Normalizer} + if err := setRepairSkillSource(&pkg, installation, rev); err != nil { + return nil, err } if installation.Source.SourceKind == "archive" { digest := strings.TrimPrefix(rev.ResolvedRevision, "sha256:") diff --git a/cli/internal/agentpluginscli/deps_repair_source.go b/cli/internal/agentpluginscli/deps_repair_source.go new file mode 100644 index 000000000..82ef58e82 --- /dev/null +++ b/cli/internal/agentpluginscli/deps_repair_source.go @@ -0,0 +1,31 @@ +package agentpluginscli + +import ( + "errors" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" +) + +// Each binding's recorded revision owns repair provenance, independently of a +// newer environment lock or another target's successful update. +func setRepairSkillSource(pkg *environment.LockedPackage, installation domain.Installation, rev *domain.ClientPackageRevision) error { + pkg.Source.Git = &environment.LockedGitSource{Repository: installation.Source.Repository, URL: installation.Source.SourceURL, Path: installation.Source.PackageSubpath, Commit: rev.ResolvedRevision, Requested: environment.Selector{Kind: environment.SelectorCommit, Value: rev.ResolvedRevision}} + if installation.Source.StableSourceKey != "" && installation.Source.SourceKind != "https-skill" { + facts := rev.Skill + if facts == nil { + return errors.New("recorded Skill source provenance unavailable; restore the matching state/lock before repair") + } + pkg.Source.Git = &environment.LockedGitSource{Kind: environment.SkillKind, Skill: facts.Name, Repository: installation.Source.Repository, Path: facts.RequestedPath, Commit: rev.ResolvedRevision, Requested: environment.Selector{Kind: environment.SelectorCommit, Value: rev.ResolvedRevision}, ResolvedSkillPath: facts.ResolvedPath, RawTreeDigest: facts.RawTreeDigest, Normalizer: facts.Normalizer} + } + if installation.Source.SourceKind == "https-skill" { + if rev.WebSkill == nil { + return errors.New("recorded HTTPS Skill provenance unavailable; restore matching state/lock before repair") + } + pkg.Source.Git = nil + facts := *rev.WebSkill + facts.Files = append([]domain.WebSkillFile{}, rev.WebSkill.Files...) + pkg.Source.Web = &facts + } + return nil +} diff --git a/cli/internal/agentpluginscli/deps_resolver.go b/cli/internal/agentpluginscli/deps_resolver.go index acafb6b9b..7e5c280e1 100644 --- a/cli/internal/agentpluginscli/deps_resolver.go +++ b/cli/internal/agentpluginscli/deps_resolver.go @@ -76,6 +76,9 @@ func (resolver dependencyResolver) Resolve(ctx context.Context, requirement envi if previous != nil && !update && sameRequestedSelector(requirement, *previous) { return *previous, nil } + if requirement.Web != nil { + return resolver.resolveWebSkill(ctx, requirement, targets) + } if requirement.Git != nil { return resolver.resolveGit(ctx, requirement, targets, previous, update) } @@ -217,7 +220,12 @@ func (resolver dependencyResolver) finishLoaded(ctx context.Context, loaded load _ = loaded.cleanup() return err } - key := dependencySnapshotKey{sourceKey: pin.SourceKey, revision: pin.Revision(), algorithm: pin.DigestAlgorithm, digest: pin.TreeDigest} + key, err := lockedDependencySnapshotKey(pin) + if err != nil { + _ = packagedigest.Remove(snapshot) + _ = loaded.cleanup() + return err + } cache := resolver.app.dependencySnapshots if old, ok := cache.snapshots[key]; ok { _ = packagedigest.Remove(old) diff --git a/cli/internal/agentpluginscli/deps_source.go b/cli/internal/agentpluginscli/deps_source.go index 2679844ba..60d9e4d6f 100644 --- a/cli/internal/agentpluginscli/deps_source.go +++ b/cli/internal/agentpluginscli/deps_source.go @@ -32,6 +32,9 @@ func legacyGitIdentity(repository, commit, packagePath, hint string) domain.Sour } func identityFromLocked(pin environment.LockedPackage) domain.SourceIdentity { + if pin.Source.Web != nil { + return webSkillSnapshotSource(pin) + } if source := pin.Source.Git; source != nil { if source.Kind == environment.SkillKind { return skillSnapshotSource(pin) @@ -68,6 +71,9 @@ func (app App) acquireDependencyGit(ctx context.Context, source environment.GitS } func (app App) acquirePinnedSource(ctx context.Context, pin environment.LockedPackage) (domain.PackageSnapshot, error) { + if pin.Source.Web != nil { + return app.acquireLockedWebSkill(ctx, pin) + } if source := pin.Source.Git; source != nil { if source.Kind == environment.SkillKind { return app.acquireLockedSkill(ctx, pin) diff --git a/cli/internal/agentpluginscli/deps_staging.go b/cli/internal/agentpluginscli/deps_staging.go index f043e7be3..2e7db37ac 100644 --- a/cli/internal/agentpluginscli/deps_staging.go +++ b/cli/internal/agentpluginscli/deps_staging.go @@ -2,6 +2,9 @@ package agentpluginscli import ( "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" "errors" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" @@ -14,19 +17,6 @@ type dependencySnapshotKey struct { kind, skill, requestedPath, rawDigest, normalizer string } -func lockedDependencyCoordinates(locked environment.LockedPackage) (string, string, string, error) { - if source := locked.Source.Git; source != nil { - if source.Kind == environment.SkillKind { - return source.Repository, source.Commit, source.ResolvedSkillPath, nil - } - return source.Repository, source.Commit, source.Path, nil - } - if source := locked.Source.Registry; source != nil { - return source.Repository, source.Commit, source.Path, nil - } - return "", "", "", errors.New("locked package has no source") -} - // The command owns these acquired bytes until its final preflight/apply returns. // Each caller gets a fresh verified snapshot and performs its own loading, // scanning and Directory policy checks; eligibility is never cached here. @@ -42,7 +32,7 @@ func (cache *dependencySnapshotCache) close() { } func lockedDependencySnapshotKey(locked environment.LockedPackage) (dependencySnapshotKey, error) { - if locked.Source.Git == nil && locked.Source.Registry == nil && locked.Source.Archive == nil { + if locked.Source.Git == nil && locked.Source.Registry == nil && locked.Source.Archive == nil && locked.Source.Web == nil { return dependencySnapshotKey{}, errors.New("locked package has no source") } sourceKey := locked.SourceKey @@ -54,6 +44,16 @@ func lockedDependencySnapshotKey(locked environment.LockedPackage) (dependencySn key.kind, key.skill, key.requestedPath = source.Kind, source.Skill, source.Path key.rawDigest, key.normalizer = source.RawTreeDigest, source.Normalizer } + if source := locked.Source.Web; source != nil { + key.kind, key.skill = "https-skill", source.Skill + key.rawDigest, key.normalizer = source.RawTreeDigest, source.Normalizer + body, err := json.Marshal(source) + if err != nil { + return dependencySnapshotKey{}, err + } + digest := sha256.Sum256(body) + key.requestedPath = hex.EncodeToString(digest[:]) + } return key, nil } @@ -102,7 +102,7 @@ func (app App) acquireLockedSnapshot(ctx context.Context, locked environment.Loc } func lockedDependencySource(locked environment.LockedPackage) (domain.SourceIdentity, error) { - if locked.Source.Git == nil && locked.Source.Registry == nil && locked.Source.Archive == nil { + if locked.Source.Git == nil && locked.Source.Registry == nil && locked.Source.Archive == nil && locked.Source.Web == nil { return domain.SourceIdentity{}, errors.New("locked package has no source") } return identityFromLocked(locked), nil diff --git a/cli/internal/agentpluginscli/deps_sync.go b/cli/internal/agentpluginscli/deps_sync.go index 6eae770ee..8c315ccc1 100644 --- a/cli/internal/agentpluginscli/deps_sync.go +++ b/cli/internal/agentpluginscli/deps_sync.go @@ -72,6 +72,9 @@ func materializeLockedPackage(ctx context.Context, app App, locked environment.L } func dependencySourceHint(locked environment.LockedPackage) string { + if locked.Source.Web != nil { + return "uap-dependency-web-skill" + } if locked.Source.Git != nil && locked.Source.Git.Kind == environment.SkillKind { return "uap-dependency-skill" } diff --git a/cli/internal/agentpluginscli/deps_web_skills.go b/cli/internal/agentpluginscli/deps_web_skills.go new file mode 100644 index 000000000..ecd564c01 --- /dev/null +++ b/cli/internal/agentpluginscli/deps_web_skills.go @@ -0,0 +1,77 @@ +package agentpluginscli + +import ( + "context" + "errors" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/skillsource" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" +) + +type webSkillAcquirer interface { + ResolveHTTPSkill(context.Context, string, string) (domain.PackageSnapshot, domain.WebSkillSourceFacts, error) + AcquireHTTPSkillVerified(context.Context, domain.WebSkillSourceFacts) (domain.PackageSnapshot, error) +} + +func (app App) webSkills() (webSkillAcquirer, error) { + a, ok := app.SourceAcquirer.(webSkillAcquirer) + if !ok { + return nil, errors.New("HTTPS Skill acquisition unavailable") + } + return a, nil +} +func (resolver dependencyResolver) resolveWebSkill(ctx context.Context, req environment.Requirement, targets []string) (environment.LockedPackage, error) { + a, err := resolver.app.webSkills() + if err != nil { + return environment.LockedPackage{}, err + } + raw, facts, err := a.ResolveHTTPSkill(ctx, req.Web.URL, req.Web.Skill) + if err != nil { + return environment.LockedPackage{}, err + } + defer func() { _ = packagedigest.Remove(raw) }() + snapshot, err := skillsource.Normalize(ctx, raw, req.Web.Skill, mustSourceKey(req)) + if err != nil { + return environment.LockedPackage{}, err + } + facts.RawTreeDigest, facts.Normalizer = raw.TreeDigest, environment.SkillNormalizerV1 + pin := environment.LockedPackage{Alias: req.Alias, SourceKey: mustSourceKey(req), Targets: append([]string{}, targets...), DigestAlgorithm: snapshot.DigestAlgorithm, TreeDigest: snapshot.TreeDigest, ExecutableFiles: snapshot.ExecutableFiles, Source: environment.LockedSource{Web: &facts}} + snapshot.Source = webSkillSnapshotSource(pin) + loaded, err := resolver.app.loadStandardSnapshot(ctx, snapshot, domain.OriginModeDirect, nil, nil) + if err != nil { + return environment.LockedPackage{}, err + } + pin.ManifestDigest = loaded.envelope.ManifestDigest + return pin, resolver.finishLoaded(ctx, loaded, pin) +} +func webSkillSnapshotSource(pin environment.LockedPackage) domain.SourceIdentity { + facts := *pin.Source.Web + facts.Files = append([]domain.WebSkillFile{}, facts.Files...) + return domain.SourceIdentity{SourceKind: "https-skill", SourceURL: facts.URL, WebSkill: &facts, StableSourceKey: pin.SourceKey, RequestedSource: pin.SourceKey, CanonicalSource: pin.SourceKey, ResolvedRevision: facts.RawTreeDigest, SourceBindingHint: "uap-dependency-web-skill"} +} +func (app App) acquireLockedWebSkill(ctx context.Context, pin environment.LockedPackage) (domain.PackageSnapshot, error) { + if pin.Source.Web.Normalizer != environment.SkillNormalizerV1 { + return domain.PackageSnapshot{}, errors.New("unsupported locked HTTPS Skill normalizer") + } + a, err := app.webSkills() + if err != nil { + return domain.PackageSnapshot{}, err + } + raw, err := a.AcquireHTTPSkillVerified(ctx, *pin.Source.Web) + if err != nil { + return domain.PackageSnapshot{}, err + } + defer func() { _ = packagedigest.Remove(raw) }() + snapshot, err := skillsource.Normalize(ctx, raw, pin.Source.Web.Skill, pin.SourceKey) + if err != nil { + return domain.PackageSnapshot{}, err + } + if snapshot.TreeDigest != pin.TreeDigest { + _ = packagedigest.Remove(snapshot) + return domain.PackageSnapshot{}, errors.New("normalized HTTPS Skill artifact differs from uap.lock") + } + snapshot.Source = webSkillSnapshotSource(pin) + return snapshot, nil +} diff --git a/cli/internal/agentpluginscli/environment_session.go b/cli/internal/agentpluginscli/environment_session.go index 2c90e8b9f..50fec1c31 100644 --- a/cli/internal/agentpluginscli/environment_session.go +++ b/cli/internal/agentpluginscli/environment_session.go @@ -239,7 +239,9 @@ func dependencyIntentEdits(previous, wanted environment.Requirements) []manifest edit := manifestedit.DependencyEdit{Alias: alias} if ok { v := &manifestedit.Dependency{} - if value.Git != nil { + if value.Web != nil { + v.Website, v.Kind, v.Skill = value.Web.URL, environment.SkillKind, value.Web.Skill + } else if value.Git != nil { v.Git = value.Git.Endpoint() v.Path = value.Git.Path v.Kind, v.Skill = value.Git.Kind, value.Git.Skill diff --git a/cli/internal/agentpluginscli/manifestedit/edit.go b/cli/internal/agentpluginscli/manifestedit/edit.go index a5925a8e8..b0d9e0fc2 100644 --- a/cli/internal/agentpluginscli/manifestedit/edit.go +++ b/cli/internal/agentpluginscli/manifestedit/edit.go @@ -21,7 +21,7 @@ var ErrUnsupportedLayout = errors.New("unsupported TOML layout; edit uap.toml ma // Empty fields are omitted. Release is an exact sequence represented as a TOML // string; semantic acceptance remains the strict manifest reader's job. type Dependency struct { - Git, Path, Ref, Commit, Registry, Distribution, Release, Version, Archive, SHA256, Kind, Skill string + Git, Path, Ref, Commit, Registry, Distribution, Release, Version, Archive, SHA256, Kind, Skill, Website string } type DependencyEdit struct { @@ -116,7 +116,7 @@ func editDependency(doc *tomledit.Document, edit DependencyEdit) error { section = appendSection(doc, name) } fields := []struct{ name, value string }{ - {"kind", edit.Value.Kind}, {"skill", edit.Value.Skill}, {"git", edit.Value.Git}, {"archive", edit.Value.Archive}, {"sha256", edit.Value.SHA256}, {"path", edit.Value.Path}, {"ref", edit.Value.Ref}, + {"website", edit.Value.Website}, {"kind", edit.Value.Kind}, {"skill", edit.Value.Skill}, {"git", edit.Value.Git}, {"archive", edit.Value.Archive}, {"sha256", edit.Value.SHA256}, {"path", edit.Value.Path}, {"ref", edit.Value.Ref}, {"commit", edit.Value.Commit}, {"registry", edit.Value.Registry}, {"distribution", edit.Value.Distribution}, {"release", edit.Value.Release}, } diff --git a/cli/internal/agentpluginscli/manifestedit/edit_test.go b/cli/internal/agentpluginscli/manifestedit/edit_test.go index 1a291cda7..307ac993d 100644 --- a/cli/internal/agentpluginscli/manifestedit/edit_test.go +++ b/cli/internal/agentpluginscli/manifestedit/edit_test.go @@ -191,3 +191,24 @@ func TestApplyCanonicalKeysPreserveIntentCommentsAndCRLF(t *testing.T) { t.Fatalf("lost CRLF: %q", out) } } + +// Replacing website intent must remove stale transport fields in both directions. +func TestWebsiteSkillReplacement(t *testing.T) { + input := []byte("schema_version=1\n[dependencies.pdf]\ngit='acme/tools'\nref='main'\n[targets]\ncodex=['pdf']\n") + out, err := Apply(input, Changes{Dependencies: []DependencyEdit{{Alias: "pdf", Value: &Dependency{Website: "https://publisher.example", Kind: "skill", Skill: "pdf"}}}}) + if err != nil { + t.Fatal(err) + } + deps := decode(t, out)["dependencies"].(map[string]any)["pdf"].(map[string]any) + if deps["website"] != "https://publisher.example" || deps["kind"] != "skill" || deps["skill"] != "pdf" || deps["git"] != nil || deps["ref"] != nil { + t.Fatal(deps) + } + out, err = Apply(out, Changes{Dependencies: []DependencyEdit{{Alias: "pdf", Value: &Dependency{Git: "acme/tools", Ref: "main"}}}}) + if err != nil { + t.Fatal(err) + } + deps = decode(t, out)["dependencies"].(map[string]any)["pdf"].(map[string]any) + if deps["website"] != nil || deps["kind"] != nil || deps["skill"] != nil || deps["git"] != "acme/tools" { + t.Fatal(deps) + } +} diff --git a/cli/internal/agentpluginscli/skills.go b/cli/internal/agentpluginscli/skills.go index 2191ecef0..8735981d7 100644 --- a/cli/internal/agentpluginscli/skills.go +++ b/cli/internal/agentpluginscli/skills.go @@ -21,9 +21,9 @@ type skillsAddOptions struct { } func newSkillsCommand(app App, opts *options) *cobra.Command { - root := &cobra.Command{Use: "skills", Short: "Install raw Agent Skills from public GitHub sources"} + root := &cobra.Command{Use: "skills", Short: "Install raw Agent Skills from GitHub or HTTPS publisher indexes"} selection := &skillsAddOptions{} - add := &cobra.Command{Use: "add ", Short: "Select and install raw Skills through the saved environment installer", Long: "Select Skills from a public GitHub repository, folder or SKILL.md URL.\n\nExplicit --skill, --path, --target and exact file/folder URLs use terminal review.\n--ui opens interactive selection and cannot combine with --skill, --target or --dry-run.\nJSON and non-interactive use require explicit selections, targets and --yes (except --dry-run).\nTerminal selection: numbers toggle Skills, /query searches, / clears the filter,\nclear removes all selections, done reviews, q or Enter cancels.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return selection.run(cmd, app, opts, args[0]) }} + add := &cobra.Command{Use: "add ", Short: "Select and install raw Skills through the saved environment installer", Long: "Select Skills from a public GitHub repository, folder or SKILL.md URL.\nHTTPS publishers expose a complete .well-known Skills index; use --skill with an exact name.\n\nExplicit --skill, --path, --target and exact file/folder URLs use terminal review.\n--ui opens interactive selection and cannot combine with --skill, --target or --dry-run.\nJSON and non-interactive use require explicit selections, targets and --yes (except --dry-run).\nTerminal selection: numbers toggle Skills, /query searches, / clears the filter,\nclear removes all selections, done reviews, q or Enter cancels.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return selection.run(cmd, app, opts, args[0]) }} add.Flags().StringSliceVar(&selection.names, "skill", nil, "exact Skill name (repeatable; ambiguity requires --path)") add.Flags().StringVar(&selection.folder, "path", "", "exact repository-relative Skill folder (empty selects repository root)") add.Flags().StringVar(&selection.ref, "ref", "", "exact Git ref when a URL boundary is ambiguous") @@ -63,6 +63,9 @@ func (selection *skillsAddOptions) run(cmd *cobra.Command, app App, opts *option if err := cmd.Context().Err(); err != nil { return err } + if isWebsiteSkillInput(input) { + return selection.runWebsite(cmd, app, opts, input, interactive) + } request := SkillsPickerRequest{Input: input, Ref: selection.ref, Path: selection.folder, PathSet: cmd.Flags().Changed("path")} if request.PathSet { if err := sourceacquisition.ValidateSkillPath(request.Path); err != nil { diff --git a/cli/internal/agentpluginscli/skills_batch.go b/cli/internal/agentpluginscli/skills_batch.go index 0516b5001..89f4aff70 100644 --- a/cli/internal/agentpluginscli/skills_batch.go +++ b/cli/internal/agentpluginscli/skills_batch.go @@ -16,6 +16,7 @@ var ErrSkillsPlanStale = errors.New("skill installation plan is stale") type SkillsSelection struct { Repository, Commit, Path, Name, Alias, CandidateID string + Website string Selector environment.Selector } type SkillsBatchRequest struct { diff --git a/cli/internal/agentpluginscli/skills_batch_selection.go b/cli/internal/agentpluginscli/skills_batch_selection.go index 4a5d6185f..82d10cf1a 100644 --- a/cli/internal/agentpluginscli/skills_batch_selection.go +++ b/cli/internal/agentpluginscli/skills_batch_selection.go @@ -47,7 +47,14 @@ func (candidate *skillsBatchCandidate) selectSkill(ctx context.Context, app App, if old, ok := candidate.requirements.Dependencies[alias]; ok && mustSourceKey(old) == mustSourceKey(req) { req = old } else { - pin, err := (dependencyResolver{app: app, requirements: candidate.requirements, dryRun: true}).resolveSkill(ctx, req, targets, nil, selection.Commit, "", "") + resolver := dependencyResolver{app: app, requirements: candidate.requirements, dryRun: true} + var pin environment.LockedPackage + var err error + if req.Web != nil { + pin, err = resolver.resolveWebSkill(ctx, req, targets) + } else { + pin, err = resolver.resolveSkill(ctx, req, targets, nil, selection.Commit, "", "") + } if err != nil { return err } @@ -70,6 +77,14 @@ func skillSelectionRequirement(wanted environment.Requirements, selection Skills selector = environment.Selector{Kind: environment.SelectorCommit, Value: selection.Commit} } req := environment.Requirement{Alias: defaultSkillAlias(selection), Git: &environment.GitSource{Kind: environment.SkillKind, Skill: selection.Name, Repository: strings.ToLower(selection.Repository), Path: selection.Path, Selector: selector}} + if selection.Website != "" { + req.Git = nil + endpoint, err := environment.NormalizeWebURL(selection.Website) + if err != nil { + return req, err + } + req.Web = &environment.WebSource{URL: endpoint, Skill: selection.Name} + } key, err := req.SourceKey() if err != nil { return req, err @@ -105,6 +120,10 @@ func defaultSkillAlias(selection SkillsSelection) string { return selection.Alias } probe := environment.Requirement{Alias: selection.Name, Git: &environment.GitSource{Repository: selection.Repository, Selector: environment.Selector{Kind: environment.SelectorDefault}}} + if selection.Website != "" { + probe.Git = nil + probe.Web = &environment.WebSource{URL: selection.Website, Skill: selection.Name} + } if probe.Validate() == nil { return selection.Name } diff --git a/cli/internal/agentpluginscli/skills_website.go b/cli/internal/agentpluginscli/skills_website.go new file mode 100644 index 000000000..fb5b2f970 --- /dev/null +++ b/cli/internal/agentpluginscli/skills_website.go @@ -0,0 +1,47 @@ +package agentpluginscli + +import ( + "errors" + "net/url" + "strings" + + "github.com/spf13/cobra" + + "github.com/777genius/plugin-kit-ai/cli/internal/agentpluginscli/skillspicker" +) + +// Website discovery is explicit and deterministic: a publisher index and exact +// names, never HTML crawling or heuristics over arbitrary website content. +func (selection *skillsAddOptions) runWebsite(cmd *cobra.Command, app App, opts *options, endpoint string, interactive bool) error { + if selection.ui || selection.ref != "" || cmd.Flags().Changed("path") { + return errors.New("HTTPS Skills use --skill names; --ui, --ref and --path apply only to Git sources") + } + if len(selection.names) == 0 { + return skillsMissingSelection(cmd, opts, "SELECTION_REQUIRED", "HTTPS Skill sources require an exact --skill name", []skillspicker.Candidate{}) + } + targets, err := skillsTerminalTargets(cmd, app, opts, interactive) + if err != nil { + return err + } + if !interactive && !selection.yes && !opts.dryRun { + return skillsMissingSelection(cmd, opts, "CONFIRMATION_REQUIRED", "Pass --yes to accept installation; --dry-run reviews without applying", []skillspicker.Candidate{}) + } + request := SkillsBatchRequest{} + for _, name := range selection.names { + request.Selections = append(request.Selections, SkillsSelection{Website: endpoint, Name: name}) + } + for _, target := range targets { + request.TargetIDs = append(request.TargetIDs, string(target)) + } + prepared, err := app.PrepareSkills(cmd.Context(), request) + if err != nil { + return err + } + defer prepared.Close() + return runSkillsReviewedPlan(cmd, app, opts, prepared, interactive && !selection.yes) +} + +func isWebsiteSkillInput(input string) bool { + u, err := url.Parse(input) + return err == nil && u.Scheme == "https" && !strings.EqualFold(u.Hostname(), "github.com") && !strings.EqualFold(u.Hostname(), "raw.githubusercontent.com") +} diff --git a/cli/internal/agentpluginscli/skills_website_e2e_test.go b/cli/internal/agentpluginscli/skills_website_e2e_test.go new file mode 100644 index 000000000..790ba3250 --- /dev/null +++ b/cli/internal/agentpluginscli/skills_website_e2e_test.go @@ -0,0 +1,167 @@ +package agentpluginscli + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/sourceacquisition" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/environment" +) + +// Real TLS and the full command/environment/standard loader/transaction path; +// only the native client runner and security scanner use disposable fixtures. +func TestWebsiteSkillsSavedLifecycleAndColdRefusal(t *testing.T) { + var mu sync.Mutex + version := 1 + indexAvailable := true + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + if r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" { + t.Error("publisher received credentials") + } + switch r.URL.Path { + case "/scoped/.well-known/agent-skills/index.json": + if !indexAvailable { + http.NotFound(w, r) + return + } + _, _ = fmt.Fprint(w, `{"skills":[{"name":"pdf","description":"Read PDFs","files":["SKILL.md","references/note.txt"]}]}`) + case "/scoped/.well-known/agent-skills/pdf/SKILL.md": + _, _ = fmt.Fprint(w, "---\nname: pdf\ndescription: Read PDF documents\n---\nRead references/note.txt.\n") + case "/scoped/.well-known/agent-skills/pdf/references/note.txt": + _, _ = fmt.Fprintf(w, "resource version %d", version) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + makeJourney := func() savedJourney { + j := newSavedJourney(t, t.TempDir()) + a := sourceacquisition.Acquirer{TempRoot: t.TempDir(), HTTPClient: server.Client()} + j.cli.app.SourceAcquirer = a + j.cli.app.SourceAcquirerForDryRun = nil + return j + } + j := makeJourney() + j.run(t, "skills", "add", server.URL+"/scoped", "--skill", "pdf", "--target", "codex", "--no-ui", "--yes", "--format", "json") + req, lock := j.intent(t) + pin := lock.Packages[0] + if req.Dependencies["pdf"].Web == nil || pin.Source.Web == nil || len(pin.Source.Web.Files) != 2 || pin.Source.Web.RawTreeDigest == pin.TreeDigest { + t.Fatal("website source pin missing", lock) + } + before := j.state(t) + addedBinding := savedJourneyBinding(t, before, "codex") + if got := dependencyRead(t, filepath.Join(addedBinding.TargetLocator, "skills", "pdf", "references", "note.txt")); got != "resource version 1" { + t.Fatal("installed resource lost", got) + } + repairs, err := buildDependencyRepairs(context.Background(), j.cli.app, before, before.Installations[0], []domain.ClientID{domain.ClientCodex}) + if err != nil || repairs[0].Package.Source.Web == nil { + t.Fatal("repair provenance missing", err) + } + j.run(t, "deps", "sync", "--locked", "--yes", "--format", "json") + exchange := filepath.Join(t.TempDir(), "agreement") + j.run(t, "deps", "export", exchange, "--yes", "--format", "json") + // Cold import uses recorded file coordinates even when discovery is gone. + mu.Lock() + indexAvailable = false + mu.Unlock() + cold := makeJourney() + cold.run(t, "deps", "import", exchange, "--install", "--yes", "--format", "json") + _, restored := cold.intent(t) + if restored.Packages[0].TreeDigest != pin.TreeDigest { + t.Fatal("cold import changed pin") + } + coldState := cold.state(t) + binding := savedJourneyBinding(t, coldState, "codex") + resource := filepath.Join(binding.TargetLocator, "skills", "pdf", "references", "note.txt") + if got := dependencyRead(t, resource); got != "resource version 1" { + t.Fatal("cold projection lost supporting resource", got) + } + dependencyWrite(t, filepath.Join(binding.TargetLocator, "drift.txt"), "local drift") + if err := os.RemoveAll(cold.cli.app.EnvironmentStoreRoot); err != nil { + t.Fatal(err) + } + cold.run(t, "repair", coldState.Installations[0].InstallationID, "--target", "codex", "--format", "json") + if got := dependencyRead(t, resource); got != "resource version 1" { + t.Fatal("cold repair changed pinned resource", got) + } + if _, err := os.Stat(filepath.Join(binding.TargetLocator, "drift.txt")); !os.IsNotExist(err) { + t.Fatal("repair did not replace drift", err) + } + // Explicit update follows the publisher, while the old per-client repair and + // exported agreement continue to require their original content hashes. + mu.Lock() + indexAvailable = true + version = 2 + mu.Unlock() + j.run(t, "update", "pdf", "--format", "json") + _, updated := j.intent(t) + if updated.Packages[0].TreeDigest == pin.TreeDigest || updated.Packages[0].SourceKey != pin.SourceKey { + t.Fatal("update failed to preserve identity and advance content") + } + if err = os.RemoveAll(j.cli.app.EnvironmentStoreRoot); err != nil { + t.Fatal(err) + } + savedState := dependencyRead(t, j.cli.store.Path) + _, savedLockPath := j.cli.app.dependencyPaths() + savedLock := dependencyRead(t, savedLockPath) + _, err = materializeLockedPackage(context.Background(), j.cli.app, *repairs[0].Package, false, nil) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "digest") && !strings.Contains(strings.ToLower(err.Error()), "checksum") { + t.Fatalf("old repair silently upgraded or wrong refusal: %v", err) + } + if dependencyRead(t, j.cli.store.Path) != savedState || dependencyRead(t, savedLockPath) != savedLock { + t.Fatal("historical refusal changed current state or lock") + } + refused := makeJourney() + mp, lp := refused.cli.app.dependencyPaths() + _, _, err = refused.cli.execute(false, "deps", "import", exchange, "--install", "--yes", "--format", "json") + if err == nil { + t.Fatal("changed publisher bytes accepted for historical cold import") + } + for _, p := range []string{mp, lp} { + if _, e := os.Stat(p); !os.IsNotExist(e) { + t.Fatalf("failed import published %s: %v", p, e) + } + } + if len(refused.state(t).Installations) != 0 { + t.Fatal("failed import changed profiles") + } +} + +// A staged artifact must not satisfy another pin whose file provenance differs, +// even when the raw and normalized tree digests have been copied unchanged. +func TestWebsiteSnapshotCacheIncludesFilePins(t *testing.T) { + pin := environment.LockedPackage{SourceKey: "https-skill:https://publisher.example:pdf", DigestAlgorithm: domain.TreeDigestAlgorithm, TreeDigest: "sha256:" + strings.Repeat("a", 64), Source: environment.LockedSource{Web: &environment.LockedWebSource{URL: "https://publisher.example", Skill: "pdf", IndexSHA256: strings.Repeat("b", 64), RawTreeDigest: "sha256:" + strings.Repeat("c", 64), Normalizer: environment.SkillNormalizerV1, Files: []domain.WebSkillFile{{Path: "SKILL.md", URL: "https://publisher.example/.well-known/skills/pdf/SKILL.md", SHA256: strings.Repeat("d", 64)}}}}} + original, err := lockedDependencySnapshotKey(pin) + if err != nil { + t.Fatal(err) + } + pin.Source.Web.Files[0].SHA256 = strings.Repeat("e", 64) + altered, err := lockedDependencySnapshotKey(pin) + if err != nil { + t.Fatal(err) + } + if original == altered { + t.Fatal("altered file pin can reuse a previously staged artifact") + } +} + +func TestWebsiteDispatchPreservesGitHubHostCase(t *testing.T) { + for _, input := range []string{"https://GitHub.com/acme/skills", "https://RAW.GITHUBUSERCONTENT.COM/acme/skills/main/SKILL.md", "https://github.com/acme/skills"} { + if isWebsiteSkillInput(input) { + t.Fatalf("GitHub input dispatched as website: %s", input) + } + } + if !isWebsiteSkillInput("https://publisher.example/skills") { + t.Fatal("HTTPS publisher not dispatched") + } +} diff --git a/docs/AGENT_SKILLS_INSTALLER.md b/docs/AGENT_SKILLS_INSTALLER.md index cba83565d..2589000cb 100644 --- a/docs/AGENT_SKILLS_INSTALLER.md +++ b/docs/AGENT_SKILLS_INSTALLER.md @@ -1,4 +1,4 @@ -# Installing Agent Skills from GitHub +# Installing Agent Skills This guide describes the source implementation in this change. It does not claim that these commands are available in an already published installer release. @@ -10,6 +10,18 @@ A Skill can be installed from a public GitHub repository, a folder URL, or a then uses the existing package installer and ownership/recovery rules. Installation does not execute the Skill's scripts. +Compatible HTTPS publishers can also expose complete Skills through a +`.well-known` discovery index. This source-build workflow requires the exact +Skill name and uses terminal review: + +```sh +agentplugins skills add https://publisher.example --skill review --target codex +``` + +For unattended installation, add `--yes`; use `--dry-run` to review without +applying. Website sources do not use the repository browser or Git options +`--ref` and `--path`. + ```sh # Interactive browser: nested folders, search, basket, target selection and review. agentplugins skills add owner/repository @@ -111,6 +123,40 @@ version. Existing protections for foreign files, drift and pending recovery appl Native clients may require their normal uninstall/recovery steps; follow the command's instructions rather than deleting agent configuration manually. +## Website sources + +The publisher must serve an index at +`/.well-known/agent-skills/index.json` or `/.well-known/skills/index.json`. +The supported discovery dialect supplies a `skills` array with each Skill's +`name` and complete `files` list. The installer downloads every listed resource +and validates the resulting Skill through the same normalization and package +installation pipeline. It does not scrape arbitrary web pages. Other discovery +formats, including proposed 0.2 artifact entries, are not supported yet. +Publisher downloads must use direct HTTPS addresses; this workflow refuses +redirects. + +Website intent uses its own source field: + +```toml +[dependencies.review] +website = "https://publisher.example" +kind = "skill" +skill = "review" + +[targets] +codex = ["review"] +``` + +The lock preserves the resolved file addresses, each file's SHA256, discovery +provenance, raw tree digest and normalizer version. `deps update review` explicitly +discovers a newer version. Locked sync, repair and export/import restore the +recorded bytes without selecting a newer version from the index. If the old +files have changed or disappeared and no verified cache remains, restoration +fails; the publisher must retain those bytes for cold restore to succeed. + +Website discovery remains an emerging convention. This compatibility does not +promise that all websites or every proposed discovery format can be installed. + ## Building and checking the picker From the repository root, use the pinned toolchain and package lock: diff --git a/docs/plans/source-transports-e2e/website-skills-evidence.md b/docs/plans/source-transports-e2e/website-skills-evidence.md new file mode 100644 index 000000000..753723d5d --- /dev/null +++ b/docs/plans/source-transports-e2e/website-skills-evidence.md @@ -0,0 +1,76 @@ +# Website Skill implementation evidence + +2026-10-24. Extension to PR #473, source-only and unreleased. This evidence does +not qualify native agent execution or an authoring release. + +## Lifecycle checks + +`TestWebsiteSkillsSavedLifecycleAndColdRefusal` exercises the actual public CLI +composition against a disposable TLS publisher and isolated client roots. +Only the native client runner and security scanner use existing test fixtures. +The extended scenario passed locally in 7.290 seconds: + +- Select and install one scoped website Skill, including actual nested resource + bytes, intent and immutable lock. +- Repeat locked sync and export the agreement. +- Cold import from recorded file coordinates while the discovery index is + unavailable, verifying the restored nested resource. +- Lose the store, introduce managed drift and run the public repair command + without rediscovery. +- Explicitly update to changed publisher content while retaining source identity. +- Refuse historical repair and cold import when old file bytes have changed, + preserving existing state/lock and publishing no new installation on failure. + +After the final portability and architecture corrections, root ran the focused +CLI regression with `GOTOOLCHAIN=go1.26.10`: PASS in 12.250 seconds. + +```sh +go test ./cli/internal/agentpluginscli -run 'TestSkillsBatch|TestDependencyStagingSeparatesSkill|TestSkillRootRequiresRaw|TestSourceSkills|TestWebsite' -count=1 +``` + +Final adapter package run passed in 25.787 seconds. Environment, manifest +decoder/editor, domain, installer and state-v2 suites passed. The full usecase +suite encountered `TestPublicS1GeminiFailedFirstRetriesCompleteDesiredWithoutAdoptingForeignBytes` +panicking at `public_native_retry_test.go:168` on local macOS. Root reproduced +the exact focused test with the CI Go 1.26.10 toolchain both on this change and +on a separate clean checkout of pre-extension PR head +`7aa99d5a42469e54c15ca7cf9f988bbebf9eef4b`: both panic at the same line because +the fixture has no installation. This existing local failure is not a green +full-suite result or a newly introduced website-source regression. + +Root's final Go 1.26.10 focused domain/acquisition/installer website tests passed +(0.468s / 1.131s / 1.187s). Pinned golangci-lint 2.13.2 and Go 1.26.10 report +zero issues for both CLI and agentplugins modules, both changed-line checks against +`origin/main` and whole-file size/complexity/import-boundary checks. The original +oversized acquirer helper was extracted without relaxing lint configuration. + +Read-only independent review covered source authority, exact replay, per-client +repair and portable paths. Review and implementation checks found cache +pin-fingerprint omissions, Unicode file/directory aliasing and mixed-case GitHub routing; fixes and narrow +regressions are included. Final review confirmed both fresh and verified +acquisition reject Unicode collisions before file downloads. Installation does +not execute publisher scripts. + +## Real publisher acquisition + +A temporary read-only harness called the production acquisition adapter and +normalizer over normal public HTTPS. Receipt captured at +2026-10-24T12:42:06.797255Z: + +| Publisher / Skill | Result | +| --- | --- | +| `https://uizze.com` / `ui-radar` | Complete listed tree acquired and normalized: 1 file, 4,739 bytes. Raw digest `sha256:bee69e0f63060bbf45e1ff2179b8a907c37e40706e0ac42af6ecc2e27662e150`; normalized digest `sha256:7651a2d497a8546b81826e939cd8606aa239b6651149d997eddae4d21c39ce5f`. | +| `https://open.feishu.cn` / `lark-attendance` | Preferred HTML response correctly fell back to legacy index. Complete listed tree acquired: 1 file, 1,620 bytes; raw digest `sha256:d98d2796275d09b298f1a5ee879a98c6fab0b0ab59e0fa0ac8fd58ebd3d6b333`. Existing strict normalization rejected its metadata with `metadata values must be strings`. This is an observed compatibility limit, not a successful installation. | + +Uizze index hash: +`f76981128062950aa13cbafd9ae9e1d824cf0d90f1c59628261adfcf20b960a3`. +Feishu legacy index hash: +`a800fb7803dd9295a249a3db097100c4c7b0ae320cd1deb126bc25b1c66855eb`. +No real client profile or native agent was opened by this acquisition check. + +## Delivery boundary + +Existing PR #473 and its commits remain the delivery vehicle. This extension +does not merge the PR, publish a CLI package, change the landing command to an +unsupported published `latest`, or claim every catalog entry is compatible. +Discovery artifact format 0.2 and redirects remain explicitly unsupported. diff --git a/docs/plans/source-transports-e2e/website-skills.md b/docs/plans/source-transports-e2e/website-skills.md new file mode 100644 index 000000000..9263aa4d8 --- /dev/null +++ b/docs/plans/source-transports-e2e/website-skills.md @@ -0,0 +1,75 @@ +# Website Skill sources + +Owner extension, 2026-10-24: finish website Skill installation end to end and +push into existing PR #473. This supersedes the original plan's exclusion of +website raw Skills only. Preserve the transport implementation and review +history. No merge, installer publication or release is authorized. + +## Supported source contract + +`agentplugins skills add https://publisher.example --skill exact-name` selects +one Skill from a machine-readable discovery index. It does not scrape HTML, +infer resource files from Markdown, or require GitHub. Website sources have a +distinct typed saved-source and identity namespace; existing GitHub keys and +lock semantics remain unchanged. + +Support the deployed `files[]` discovery dialect: an index has a `skills` array +whose entries identify a name and complete list of relative files. Prefer +`/.well-known/agent-skills/index.json`, falling back to +`/.well-known/skills/index.json` when the preferred path returns HTTP 404 or an +HTML response with HTTP 200. A malformed JSON index is an error. Scoped addresses must retain +their scope and never silently select from the publisher's root catalog. + +This is explicit compatibility with an emerging discovery convention, not a +claim that website discovery is a finalized Agent Skills standard. Discovery +proposal [#254](https://github.com/agentskills/agentskills/pull/254) and +[discussion #255](https://github.com/agentskills/agentskills/issues/255) were open +at research time. Proposed 0.2 artifact entries (`type`, `url`, `digest`) are +outside this slice and must produce an understandable unsupported-format error. + +Research identified real deployed catalogs at +[Feishu](https://open.feishu.cn/.well-known/skills/index.json) and +[Uizze](https://uizze.sh/.well-known/agent-skills/index.json). Feishu's preferred +path served HTML, making content-aware fallback a required interoperability +case. Discovery metadata does not certify the Skill content or its compatibility. + +## Acquisition and version invariants + +Download every declared file into an owned disposable tree. Bound index bytes, +entry/file counts, individual/total file bytes and request duration; +honor cancellation and clean partial materialization. Reject unsafe or colliding +paths, malformed source URLs, credentials and unsupported formats before client +effects. Require HTTPS and normal certificate verification for every request; +refuse redirects and do not forward credentials. An index never authorizes arbitrary +filesystem paths or executable package content. + +Feed the sealed raw tree through the existing standalone Skill normalizer, +validator, scanner and installer. Keep raw content identity separate from the +normalized plugin tree and its normalizer version. Preserve complete nested +resources; installing just SKILL.md is not successful acquisition. + +Saved intent records the website URL and selected Skill name. The lock records +the discovered index URL/hash, exact file paths/URLs/hashes, raw tree digest and +normalizer version. A locked replay retrieves recorded files and verifies their +bytes without rediscovering or substituting the publisher's current selection. +Normal shared-store reuse still verifies source authority and content identity. + +Only explicit update rediscovers current publisher content and produces a new +pin. Sync, repair, export/import and cold restore preserve the recorded pin; +changed or missing old files fail without replacing the lock or active content. +The lock does not promise historical hosting when a publisher deletes old bytes. + +## Required evidence + +Use a disposable TLS publisher and disposable client roots through the public +CLI composition. Install a selected Skill with nested resources, save intent +and lock, verify projection and repeated sync, change publisher content, prove +locked replay refuses replacement, explicitly update, and restore/export/import +the exact pinned version. Exercise missing historical bytes and no-effect +failure. Keep native agent launch and runtime provisioning outside these tests. + +Adapter contract tests cover malformed/HTML preferred-index fallback, selection +ambiguity, scoped discovery, traversal/collisions, TLS/redirect policy, resource +limits, cancellation and digest mismatch. Existing GitHub Skill and plugin +transport lifecycle suites must remain green. Record actual commands and +results after implementation; this document alone is not completion evidence. diff --git a/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go b/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go index e1aafe8e5..08d6bdb19 100644 --- a/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go +++ b/install/integrationctl/agentplugins/adapters/dependencymanifest/decode.go @@ -15,6 +15,7 @@ import ( ) type rawDependency struct { + Website string `toml:"website"` Archive string `toml:"archive"` SHA256 string `toml:"sha256"` Kind string `toml:"kind"` @@ -164,6 +165,10 @@ func (doc Document) clone() environment.Requirements { source := *value.Archive value.Archive = &source } + if value.Web != nil { + source := *value.Web + value.Web = &source + } if value.Registry != nil { source := *value.Registry value.Registry = &source @@ -194,8 +199,8 @@ func decodeRawDependency(value any) (rawDependency, error) { } func requirementFromRaw(alias string, raw rawDependency) (environment.Requirement, error) { - if (raw.Kind != "" || raw.Skill != "") && raw.Git == "" { - return environment.Requirement{}, errors.New("kind and skill require a git source") + if (raw.Kind != "" || raw.Skill != "") && raw.Git == "" && raw.Website == "" { + return environment.Requirement{}, errors.New("kind and skill require a git or website source") } if raw.Version != "" { return versionRequirement(alias, raw) @@ -205,7 +210,10 @@ func requirementFromRaw(alias string, raw rawDependency) (environment.Requiremen return environment.Requirement{}, err } if rawSourceCount(raw) != 1 { - return environment.Requirement{}, errors.New("dependency must contain exactly one git, archive or registry source") + return environment.Requirement{}, errors.New("dependency must contain exactly one git, archive, registry or website source") + } + if raw.Website != "" { + return webRequirement(alias, raw) } if raw.Archive != "" { return archiveRequirement(alias, raw) @@ -221,7 +229,7 @@ func requirementFromRaw(alias string, raw rawDependency) (environment.Requiremen func rawSourceCount(raw rawDependency) int { count := 0 - for _, source := range []bool{raw.Git != "", raw.Registry != "" || raw.Distribution != "" || raw.Release != nil, raw.Archive != ""} { + for _, source := range []bool{raw.Git != "", raw.Registry != "" || raw.Distribution != "" || raw.Release != nil, raw.Archive != "", raw.Website != ""} { if source { count++ } @@ -278,6 +286,9 @@ func registryRequirement(alias string, raw rawDependency, selector environment.S } func rawSelector(raw rawDependency) (environment.Selector, error) { + if raw.Website != "" { + return environment.Selector{Kind: environment.SelectorDefault}, nil + } if raw.Version != "" { kind := environment.SelectorVersion if strings.HasPrefix(raw.Version, "^") { @@ -345,7 +356,7 @@ func validateDependencyFields(path string, body []byte, alias string, value any) } for _, key := range sortedKeys(fields) { switch key { - case "git", "path", "ref", "commit", "registry", "distribution", "archive", "sha256", "kind", "skill": + case "website", "git", "path", "ref", "commit", "registry", "distribution", "archive", "sha256", "kind", "skill": if _, ok := fields[key].(string); !ok { return manifestError(path, body, fmt.Errorf("dependencies.%s.%s must be a string", alias, key), "dependencies", alias, key) } @@ -356,3 +367,17 @@ func validateDependencyFields(path string, body []byte, alias string, value any) } return nil } + +func webRequirement(alias string, raw rawDependency) (environment.Requirement, error) { + if raw.Kind != environment.SkillKind || raw.Skill == "" { + return environment.Requirement{}, errors.New("website source requires kind=skill and an exact skill name") + } + if raw.Path != "" || raw.Ref != "" || raw.Commit != "" || raw.SHA256 != "" { + return environment.Requirement{}, errors.New("website source cannot contain Git or archive fields") + } + endpoint, err := environment.NormalizeWebURL(raw.Website) + if err != nil { + return environment.Requirement{}, err + } + return environment.Requirement{Alias: alias, Web: &environment.WebSource{URL: endpoint, Skill: raw.Skill}}, nil +} diff --git a/install/integrationctl/agentplugins/adapters/dependencymanifest/web_test.go b/install/integrationctl/agentplugins/adapters/dependencymanifest/web_test.go new file mode 100644 index 000000000..4ad2f5c4e --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/dependencymanifest/web_test.go @@ -0,0 +1,45 @@ +package dependencymanifest + +import ( + "strings" + "testing" +) + +func TestWebsiteSkillIntentDecode(t *testing.T) { + prefix := "schema_version=1\n[dependencies.pdf]\n" + doc, err := Decode([]byte(prefix+"website='https://publisher.example/prefix/'\nkind='skill'\nskill='pdf'\n[targets]\ncodex=['pdf']\n"), "uap.toml") + if err != nil { + t.Fatal(err) + } + reqs, err := doc.Root() + if err != nil { + t.Fatal(err) + } + if reqs.Dependencies["pdf"].Web.URL != "https://publisher.example/prefix" { + t.Fatal(reqs) + } + reqs.Dependencies["pdf"].Web.Skill = "other" + isolated, err := doc.Package() + if err != nil || isolated["pdf"].Web.Skill != "pdf" { + t.Fatal("document source aliases returned view", err) + } + for _, fields := range []string{ + "website='https://publisher.example'\nskill='pdf'", + "website='https://publisher.example'\nkind='skill'", + "website='https://publisher.example'\nkind='plugin'\nskill='pdf'", + "website='http://publisher.example'\nkind='skill'\nskill='pdf'", + "website='https://user@publisher.example'\nkind='skill'\nskill='pdf'", + "website='https://publisher.example?x=1'\nkind='skill'\nskill='pdf'", + "website='https://publisher.example#x'\nkind='skill'\nskill='pdf'", + "website='https://publisher.example'\nkind='skill'\nskill='pdf'\ngit='acme/tools'", + "website='https://publisher.example'\nkind='skill'\nskill='pdf'\narchive='https://publisher.example/a.zip'", + "website='https://publisher.example'\nkind='skill'\nskill='pdf'\nregistry='uap'\ndistribution='acme/pdf'\nrelease=1", + "website='https://publisher.example'\nkind='skill'\nskill='pdf'\npath='skills/pdf'", + "website='https://publisher.example'\nkind='skill'\nskill='pdf'\nref='main'", + "website='https://publisher.example'\nkind='skill'\nskill='pdf'\nsha256='" + strings.Repeat("a", 64) + "'", + } { + if _, err := Decode([]byte(prefix+fields+"\n"), "uap.toml"); err == nil { + t.Fatalf("accepted malformed website intent: %s", fields) + } + } +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go index 5139473d6..33f6cb9f9 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer.go @@ -15,7 +15,6 @@ import ( "strings" "time" - "github.com/777genius/plugin-kit-ai/install/integrationctl/adapters/pathpolicy" processadapter "github.com/777genius/plugin-kit-ai/install/integrationctl/adapters/process" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" @@ -513,50 +512,3 @@ func packagePathsFromTree(tree []byte) ([]string, error) { sort.Strings(candidates) return candidates, nil } - -func normalizeSubpath(value string) (string, error) { - value = strings.TrimSpace(strings.Trim(value, "/")) - if value == "" { - return "", nil - } - if strings.Contains(value, `\`) || path.IsAbs(value) || path.Clean(value) != value { - return "", fmt.Errorf("plugin subpath must be a normalized repository-relative directory") - } - for _, segment := range strings.Split(value, "/") { - if err := pathpolicy.ValidatePortablePathSegment(segment); err != nil { - return "", fmt.Errorf("unsafe plugin subpath %q: %w", value, err) - } - } - return value, nil -} - -func (acquirer Acquirer) runner() Runner { - if acquirer.Runner != nil { - return acquirer.Runner - } - return OSRunner{} -} - -func (acquirer Acquirer) digester() packagedigest.Builder { - digester := acquirer.Digester - if digester.TempRoot == "" { - digester.TempRoot = acquirer.TempRoot - } - return digester -} - -func (acquirer Acquirer) now() time.Time { - if acquirer.Now != nil { - return acquirer.Now().UTC() - } - return time.Now().UTC() -} - -// CanonicalGitHubRepository validates the same repository grammar used by Git -// acquisition and returns GitHub's case-insensitive owner/repository identity. -func CanonicalGitHubRepository(repository string) (string, error) { - if !repositoryPattern.MatchString(repository) { - return "", fmt.Errorf("github repository must be owner/repo") - } - return strings.ToLower(repository), nil -} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer_helpers.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer_helpers.go new file mode 100644 index 000000000..1784f4bf7 --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/acquirer_helpers.go @@ -0,0 +1,58 @@ +package sourceacquisition + +import ( + "fmt" + "path" + "strings" + "time" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/adapters/pathpolicy" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" +) + +func normalizeSubpath(value string) (string, error) { + value = strings.TrimSpace(strings.Trim(value, "/")) + if value == "" { + return "", nil + } + if strings.Contains(value, `\`) || path.IsAbs(value) || path.Clean(value) != value { + return "", fmt.Errorf("plugin subpath must be a normalized repository-relative directory") + } + for _, segment := range strings.Split(value, "/") { + if err := pathpolicy.ValidatePortablePathSegment(segment); err != nil { + return "", fmt.Errorf("unsafe plugin subpath %q: %w", value, err) + } + } + return value, nil +} + +func (acquirer Acquirer) runner() Runner { + if acquirer.Runner != nil { + return acquirer.Runner + } + return OSRunner{} +} + +func (acquirer Acquirer) digester() packagedigest.Builder { + digester := acquirer.Digester + if digester.TempRoot == "" { + digester.TempRoot = acquirer.TempRoot + } + return digester +} + +func (acquirer Acquirer) now() time.Time { + if acquirer.Now != nil { + return acquirer.Now().UTC() + } + return time.Now().UTC() +} + +// CanonicalGitHubRepository validates the same repository grammar used by Git +// acquisition and returns GitHub's case-insensitive owner/repository identity. +func CanonicalGitHubRepository(repository string) (string, error) { + if !repositoryPattern.MatchString(repository) { + return "", fmt.Errorf("github repository must be owner/repo") + } + return strings.ToLower(repository), nil +} diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web.go new file mode 100644 index 000000000..0a3d15ce4 --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web.go @@ -0,0 +1,195 @@ +package sourceacquisition + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" +) + +const ( + webSkillIndexBytes = int64(1 << 20) + webSkillFileBytes = int64(8 << 20) + webSkillTreeBytes = int64(32 << 20) + webSkillTimeout = 30 * time.Second +) + +// ResolveHTTPSkill discovers exactly one named Skill and retains every listed +// file. Its sealed snapshot contains the raw Skill, without generated metadata. +// The caller owns the snapshot through packagedigest.Remove. +func (acquirer Acquirer) ResolveHTTPSkill(ctx context.Context, endpoint, exactName string) (domain.PackageSnapshot, domain.WebSkillSourceFacts, error) { + var facts domain.WebSkillSourceFacts + base, err := domain.NormalizeWebSkillURL(endpoint) + if err != nil { + return domain.PackageSnapshot{}, facts, err + } + ctx, cancel := context.WithTimeout(ctx, webSkillTimeout) + defer cancel() + indexURL := base + "/.well-known/agent-skills/index.json" + body, status, contentType, err := acquirer.webSkillGET(ctx, indexURL, webSkillIndexBytes) + if err == nil && (status == http.StatusNotFound || status == http.StatusOK && webSkillHTML(body, contentType)) { + indexURL = base + "/.well-known/skills/index.json" + body, status, _, err = acquirer.webSkillGET(ctx, indexURL, webSkillIndexBytes) + } + if err != nil { + return domain.PackageSnapshot{}, facts, err + } + if status != http.StatusOK { + return domain.PackageSnapshot{}, facts, fmt.Errorf("skill index returned HTTP %d", status) + } + selected, err := webSkillIndexFiles(body, exactName) + if err != nil { + return domain.PackageSnapshot{}, facts, err + } + facts = domain.WebSkillSourceFacts{URL: base, Skill: exactName, IndexURL: indexURL, IndexSHA256: webSkillSHA256(body), Normalizer: "agent-skill-package/v1", RawTreeDigest: "sha256:" + strings.Repeat("0", 64)} + indexCoordinate, _ := url.Parse(indexURL) + for _, relative := range selected { + fileURL := *indexCoordinate + fileURL.Path = path.Dir(indexCoordinate.Path) + "/" + exactName + "/" + relative + fileURL.RawPath = "" + facts.Files = append(facts.Files, domain.WebSkillFile{Path: relative, URL: fileURL.String(), SHA256: strings.Repeat("0", 64)}) + } + sort.Slice(facts.Files, func(i, j int) bool { return facts.Files[i].Path < facts.Files[j].Path }) + if err := domain.ValidateWebSkillSourceFacts(facts); err != nil { + return domain.PackageSnapshot{}, domain.WebSkillSourceFacts{}, err + } + if err := validateWebSkillTreePaths(facts.Files); err != nil { + return domain.PackageSnapshot{}, domain.WebSkillSourceFacts{}, err + } + snapshot, err := acquirer.webSkillRetain(ctx, &facts, false) + if err != nil { + return domain.PackageSnapshot{}, domain.WebSkillSourceFacts{}, err + } + facts.RawTreeDigest = snapshot.TreeDigest + return snapshot, facts, nil +} + +// AcquireHTTPSkillVerified restores exact pinned file coordinates. It never +// re-fetches or resolves the mutable discovery index. +func (acquirer Acquirer) AcquireHTTPSkillVerified(ctx context.Context, facts domain.WebSkillSourceFacts) (domain.PackageSnapshot, error) { + if err := domain.ValidateWebSkillSourceFacts(facts); err != nil { + return domain.PackageSnapshot{}, err + } + if err := validateWebSkillTreePaths(facts.Files); err != nil { + return domain.PackageSnapshot{}, err + } + // Avoid changing a caller-owned files slice while checking retained pins. + facts.Files = append([]domain.WebSkillFile(nil), facts.Files...) + ctx, cancel := context.WithTimeout(ctx, webSkillTimeout) + defer cancel() + return acquirer.webSkillRetain(ctx, &facts, true) +} + +func (acquirer Acquirer) webSkillRetain(ctx context.Context, facts *domain.WebSkillSourceFacts, verify bool) (domain.PackageSnapshot, error) { + workspace, err := os.MkdirTemp(acquirer.TempRoot, "agentplugins-web-skill-*") + if err != nil { + return domain.PackageSnapshot{}, errors.New("create private Skill workspace") + } + defer func() { _ = os.RemoveAll(workspace) }() + var total int64 + for i, file := range facts.Files { + body, status, _, err := acquirer.webSkillGET(ctx, file.URL, min(webSkillFileBytes, webSkillTreeBytes-total)) + if err != nil { + return domain.PackageSnapshot{}, err + } + if status != http.StatusOK { + return domain.PackageSnapshot{}, fmt.Errorf("skill file returned HTTP %d", status) + } + total += int64(len(body)) + checksum := webSkillSHA256(body) + if verify && checksum != file.SHA256 { + return domain.PackageSnapshot{}, errors.New("skill file original-byte SHA256 does not match its locked checksum") + } + facts.Files[i].SHA256 = checksum + filename := filepath.Join(workspace, filepath.FromSlash(file.Path)) + if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil { + return domain.PackageSnapshot{}, errors.New("create private Skill file parent") + } + if err := os.WriteFile(filename, body, 0o600); err != nil { + return domain.PackageSnapshot{}, errors.New("write private Skill file") + } + } + snapshot, err := acquirer.digester().SnapshotWithExecutables(ctx, workspace, domain.SourceIdentity{}, nil) + if err != nil { + return domain.PackageSnapshot{}, fmt.Errorf("seal acquired Skill tree: %w", err) + } + if verify && snapshot.TreeDigest != facts.RawTreeDigest { + _ = packagedigest.Remove(snapshot) + return domain.PackageSnapshot{}, errors.New("skill raw tree digest does not match its locked digest") + } + facts.RawTreeDigest = snapshot.TreeDigest + key := "https-skill:" + facts.URL + ":" + facts.Skill + snapshot.Source = domain.SourceIdentity{ + SourceKind: "https-skill", SourceURL: facts.URL, WebSkill: domain.CloneWebSkillSourceFacts(facts), + StableSourceKey: key, RequestedSource: key, CanonicalSource: key, + ResolvedRevision: snapshot.TreeDigest, SourceBindingHint: "uap-dependency-web-skill", + } + snapshot.AcquiredAt = acquirer.now() + return snapshot, nil +} + +func webSkillSHA256(body []byte) string { + digest := sha256.Sum256(body) + return hex.EncodeToString(digest[:]) +} + +// webSkillGET inherits only the host's TLS/proxy transport. Cookies, automatic +// authentication, content encodings and all redirects are deliberately refused. +func (acquirer Acquirer) webSkillGET(ctx context.Context, endpoint string, limit int64) ([]byte, int, string, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody) + if err != nil { + return nil, 0, "", errors.New("create Skill HTTPS request") + } + request.Header.Set("Accept-Encoding", "identity") + client := acquirer.archiveHTTPClient() + defer client.CloseIdleConnections() + client.Timeout = min(client.Timeout, webSkillTimeout) + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + response, err := client.Do(request) + if err != nil { + if ctx.Err() != nil { + return nil, 0, "", ctx.Err() + } + return nil, 0, "", errors.New("skill HTTPS request failed (check TLS and connectivity)") + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return nil, response.StatusCode, response.Header.Get("Content-Type"), nil + } + if encoding := response.Header.Get("Content-Encoding"); encoding != "" && encoding != "identity" { + return nil, 0, "", errors.New("skill download refuses HTTP content encoding") + } + if response.ContentLength > limit { + return nil, 0, "", errors.New("skill download exceeds its byte limit") + } + body, err := io.ReadAll(io.LimitReader(&archiveContextReader{ctx: ctx, reader: response.Body}, limit+1)) + if err != nil { + return nil, 0, "", errors.New("skill download was incomplete") + } + if int64(len(body)) > limit { + return nil, 0, "", errors.New("skill download exceeds its byte limit") + } + return body, response.StatusCode, response.Header.Get("Content-Type"), nil +} + +// Publishers sometimes serve their ordinary HTML app at unknown paths with +// HTTP 200. This positive HTML signal permits the legacy index probe; malformed +// JSON, permission failures and all other responses remain terminal. +func webSkillHTML(body []byte, contentType string) bool { + mediaType := strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0]) + prefix := strings.ToLower(strings.TrimSpace(string(body))) + return strings.EqualFold(mediaType, "text/html") || strings.HasPrefix(prefix, "App", true}, + {"html body", "text/plain", "App", true}, + {"malformed json", "application/json", "{broken", false}, + {"unsupported json", "application/json", `{"skills":[{"name":"demo","url":"SKILL.md"}]}`, false}, + } { + t.Run(scenario.name, func(t *testing.T) { + probes := 0 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/prefix/.well-known/agent-skills/index.json": + w.Header().Set("Content-Type", scenario.contentType) + _, _ = fmt.Fprint(w, scenario.body) + case "/prefix/.well-known/skills/index.json": + probes++ + _, _ = fmt.Fprint(w, `{"skills":[{"name":"demo","files":["SKILL.md"]}]}`) + case "/prefix/.well-known/skills/demo/SKILL.md": + _, _ = fmt.Fprint(w, "inert") + default: + t.Errorf("unexpected request %s", r.URL.Path) + http.NotFound(w, r) + } + })) + defer server.Close() + snapshot, _, err := (Acquirer{TempRoot: t.TempDir(), HTTPClient: server.Client()}).ResolveHTTPSkill(context.Background(), server.URL+"/prefix", "demo") + if scenario.fallback { + if err != nil { + t.Fatal(err) + } + defer func() { _ = packagedigest.Remove(snapshot) }() + if probes != 1 { + t.Fatal("HTML fallback missing") + } + } else if err == nil || probes != 0 { + t.Fatalf("unexpected fallback=%d err=%v", probes, err) + } + }) + } +} diff --git a/install/integrationctl/agentplugins/adapters/statev2/source_key.go b/install/integrationctl/agentplugins/adapters/statev2/source_key.go index 4e2cd3c4a..b76157a1b 100644 --- a/install/integrationctl/agentplugins/adapters/statev2/source_key.go +++ b/install/integrationctl/agentplugins/adapters/statev2/source_key.go @@ -12,7 +12,7 @@ func installedSourceKey(installation domain.Installation) string { if installation.Source.StableSourceKey != "" { sourceKey = installation.Source.StableSourceKey } - if installation.Source.SourceKind != "" { + if installation.Source.SourceKind != "" && installation.Source.SourceKind != "https-skill" { sourceKey = sourcespec.Key(sourcespec.Kind(installation.Source.SourceKind), installation.Source.SourceURL, installation.Source.PackageSubpath) } if installation.Directory != nil { diff --git a/install/integrationctl/agentplugins/adapters/statev2/web_source_key_test.go b/install/integrationctl/agentplugins/adapters/statev2/web_source_key_test.go new file mode 100644 index 000000000..02f2f58f0 --- /dev/null +++ b/install/integrationctl/agentplugins/adapters/statev2/web_source_key_test.go @@ -0,0 +1,15 @@ +package statev2 + +import ( + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" +) + +func TestInstalledWebSkillKeyPreservesEnvironmentReceiptIdentity(t *testing.T) { + const key = "https-skill:https://skills.example/prefix:pdf" + installation := domain.Installation{Source: domain.SourceBinding{SourceKind: "https-skill", SourceURL: "https://skills.example/prefix", StableSourceKey: key}} + if actual := installedSourceKey(installation); actual != key { + t.Fatalf("HTTPS Skill receipt identity changed: %q", actual) + } +} diff --git a/install/integrationctl/agentplugins/domain/identity.go b/install/integrationctl/agentplugins/domain/identity.go index 71952200b..8b0305558 100644 --- a/install/integrationctl/agentplugins/domain/identity.go +++ b/install/integrationctl/agentplugins/domain/identity.go @@ -25,6 +25,11 @@ func newInstallationID(reader io.Reader) (string, error) { } func ComputeSourceBindingID(source SourceIdentity) string { + if source.SourceBindingHint == "uap-dependency-web-skill" && source.WebSkill != nil { + key := "https-skill:" + source.WebSkill.URL + ":" + source.WebSkill.Skill + sum := sha256.Sum256([]byte(key)) + return "src_" + hex.EncodeToString(sum[:12]) + } if source.SourceBindingHint == "uap-dependency-skill" && strings.HasPrefix(source.StableSourceKey, "github-skill:") { sum := sha256.Sum256([]byte(source.StableSourceKey)) return "src_" + hex.EncodeToString(sum[:12]) diff --git a/install/integrationctl/agentplugins/domain/state.go b/install/integrationctl/agentplugins/domain/state.go index 5dce52910..3e9931ada 100644 --- a/install/integrationctl/agentplugins/domain/state.go +++ b/install/integrationctl/agentplugins/domain/state.go @@ -204,14 +204,15 @@ type MutationReceipt struct { // projected into one client. Installation.Package is the latest accepted // revision, while individual clients may temporarily converge one at a time. type ClientPackageRevision struct { - Skill *SkillSourceFacts `json:"skill,omitempty"` - Version string `json:"version,omitempty"` - ResolvedRevision string `json:"resolved_revision,omitempty"` - TreeDigest string `json:"tree_digest"` - ManifestDigest string `json:"manifest_digest"` - DistributionID string `json:"distribution_id,omitempty"` - ReleaseSequence uint64 `json:"release_sequence,omitempty"` - CatalogEvidence *CatalogEvidence `json:"catalog_evidence,omitempty"` + Skill *SkillSourceFacts `json:"skill,omitempty"` + WebSkill *WebSkillSourceFacts `json:"web_skill,omitempty"` + Version string `json:"version,omitempty"` + ResolvedRevision string `json:"resolved_revision,omitempty"` + TreeDigest string `json:"tree_digest"` + ManifestDigest string `json:"manifest_digest"` + DistributionID string `json:"distribution_id,omitempty"` + ReleaseSequence uint64 `json:"release_sequence,omitempty"` + CatalogEvidence *CatalogEvidence `json:"catalog_evidence,omitempty"` } type ClientBinding struct { diff --git a/install/integrationctl/agentplugins/domain/types.go b/install/integrationctl/agentplugins/domain/types.go index b832b9115..aaadd3efd 100644 --- a/install/integrationctl/agentplugins/domain/types.go +++ b/install/integrationctl/agentplugins/domain/types.go @@ -60,17 +60,18 @@ func CloneSkillSourceFacts(f *SkillSourceFacts) *SkillSourceFacts { } type SourceIdentity struct { - SourceKind string `json:"source_kind,omitempty"` - SourceURL string `json:"source_url,omitempty"` - ArchiveSHA256 string `json:"archive_sha256,omitempty"` - Skill *SkillSourceFacts `json:"skill,omitempty"` - StableSourceKey string `json:"stable_source_key,omitempty"` - RequestedSource string `json:"requested_source,omitempty"` - CanonicalSource string `json:"canonical_source,omitempty"` - Repository string `json:"repository,omitempty"` - PackageSubpath string `json:"package_subpath,omitempty"` - ResolvedRevision string `json:"resolved_revision,omitempty"` - SourceBindingHint string `json:"source_binding_hint,omitempty"` + SourceKind string `json:"source_kind,omitempty"` + SourceURL string `json:"source_url,omitempty"` + ArchiveSHA256 string `json:"archive_sha256,omitempty"` + Skill *SkillSourceFacts `json:"skill,omitempty"` + WebSkill *WebSkillSourceFacts `json:"web_skill,omitempty"` + StableSourceKey string `json:"stable_source_key,omitempty"` + RequestedSource string `json:"requested_source,omitempty"` + CanonicalSource string `json:"canonical_source,omitempty"` + Repository string `json:"repository,omitempty"` + PackageSubpath string `json:"package_subpath,omitempty"` + ResolvedRevision string `json:"resolved_revision,omitempty"` + SourceBindingHint string `json:"source_binding_hint,omitempty"` } type Author struct { diff --git a/install/integrationctl/agentplugins/domain/web_skill.go b/install/integrationctl/agentplugins/domain/web_skill.go new file mode 100644 index 000000000..7b0ae844a --- /dev/null +++ b/install/integrationctl/agentplugins/domain/web_skill.go @@ -0,0 +1,21 @@ +package domain + +// WebSkillFile pins one complete publisher-provided file. URL is the exact +// download coordinate; Path is its portable path in the raw Skill tree. +type WebSkillFile struct { + Path string `json:"path"` + URL string `json:"url"` + SHA256 string `json:"sha256"` +} + +// WebSkillSourceFacts preserves discovery provenance and the immutable files +// needed to restore a Skill without consulting a mutable publisher index. +type WebSkillSourceFacts struct { + URL string `json:"url"` + Skill string `json:"skill"` + IndexURL string `json:"index_url"` + IndexSHA256 string `json:"index_sha256"` + Files []WebSkillFile `json:"files"` + RawTreeDigest string `json:"raw_tree_digest"` + Normalizer string `json:"normalizer"` +} diff --git a/install/integrationctl/agentplugins/domain/web_skill_policy.go b/install/integrationctl/agentplugins/domain/web_skill_policy.go new file mode 100644 index 000000000..db3dbb6c6 --- /dev/null +++ b/install/integrationctl/agentplugins/domain/web_skill_policy.go @@ -0,0 +1,155 @@ +package domain + +import ( + "fmt" + "net/url" + "path" + "regexp" + "strings" + "unicode" + "unicode/utf16" + "unicode/utf8" +) + +var webSkillSHA256 = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// NormalizeWebSkillURL preserves a publisher's path prefix while rejecting +// URL spellings whose decoding could change the discovery directory. +func NormalizeWebSkillURL(value string) (string, error) { + if !safeWebSkillURLText(value) { + return "", fmt.Errorf("unsafe HTTPS Skill endpoint") + } + u, err := url.Parse(value) + if err != nil || !webSkillURLCoordinates(u) || strings.Contains(value, "#") { + return "", fmt.Errorf("Skill endpoint requires credential-free HTTPS without query or fragment") + } + if err := validateWebSkillEndpointPath(u); err != nil { + return "", err + } + u.Host = strings.ToLower(u.Host) + u.Path = strings.TrimSuffix(u.Path, "/") + u.RawPath = "" + return u.String(), nil +} + +func safeWebSkillURLText(value string) bool { + return value != "" && len(value) <= 8192 && strings.TrimSpace(value) == value && !strings.ContainsAny(value, `\`) && !strings.ContainsFunc(value, unicode.IsControl) +} + +func webSkillURLCoordinates(u *url.URL) bool { + return u.Scheme == "https" && u.Hostname() != "" && u.User == nil && u.Opaque == "" && u.RawQuery == "" && !u.ForceQuery && u.Fragment == "" +} + +func validateWebSkillEndpointPath(u *url.URL) error { + if strings.ContainsFunc(u.Path, unicode.IsControl) || strings.ContainsAny(u.Path, `\`) { + return fmt.Errorf("unsafe HTTPS Skill path") + } + escaped := strings.ToLower(u.EscapedPath()) + if strings.Contains(escaped, "%2f") || strings.Contains(escaped, "%5c") { + return fmt.Errorf("encoded Skill path separators are forbidden") + } + segments := strings.Split(u.Path, "/") + for i, segment := range segments { + if segment == "." || segment == ".." || (segment == "" && i > 0 && i < len(segments)-1) { + return fmt.Errorf("Skill endpoint path must be normalized") + } + } + return nil +} + +func validWebSkillName(name string) bool { + if size := utf8.RuneCountInString(name); size < 1 || size > 64 || !utf8.ValidString(name) || strings.HasPrefix(name, "-") || strings.HasSuffix(name, "-") || strings.Contains(name, "--") { + return false + } + for _, character := range name { + if character != '-' && !unicode.IsLower(character) && !unicode.IsNumber(character) { + return false + } + } + return true +} + +// ValidateWebSkillFilePath validates the basic portable path shape. Acquisition +// additionally validates Unicode normalization and complete-tree collisions. +func ValidateWebSkillFilePath(value string) error { + if value == "" || len(value) > 1024 || strings.Contains(value, "%") || path.IsAbs(value) || path.Clean(value) != value || len(strings.Split(value, "/")) > DefaultMaxDepth { + return fmt.Errorf("Skill file must have a normalized portable relative path") + } + for _, segment := range strings.Split(value, "/") { + if !validWebSkillPathSegment(segment) { + return fmt.Errorf("Skill file path is not portable") + } + } + return nil +} + +func validWebSkillPathSegment(segment string) bool { + return segment != "" && segment != "." && segment != ".." && utf8.ValidString(segment) && len(utf16.Encode([]rune(segment))) <= 255 && !strings.ContainsAny(segment, `\/:*?"<>|`) && !strings.HasSuffix(segment, " ") && !strings.HasSuffix(segment, ".") && !strings.ContainsFunc(segment, unicode.IsControl) && !ReservedPhysicalStem(segment) +} + +// ValidateWebSkillSourceFacts validates immutable discovery provenance without +// contacting the publisher. Every download coordinate follows the pinned index. +func ValidateWebSkillSourceFacts(facts WebSkillSourceFacts) error { + base, err := NormalizeWebSkillURL(facts.URL) + if err != nil || base != facts.URL || !validWebSkillPinIdentity(facts) { + return fmt.Errorf("invalid HTTPS Skill source facts") + } + if facts.IndexURL != base+"/.well-known/agent-skills/index.json" && facts.IndexURL != base+"/.well-known/skills/index.json" { + return fmt.Errorf("Skill index must belong to its publisher endpoint") + } + if len(facts.Files) < 1 || len(facts.Files) > 256 { + return fmt.Errorf("Skill source must pin 1 to 256 files") + } + return validateWebSkillPinnedFiles(facts) +} + +func validWebSkillPinIdentity(facts WebSkillSourceFacts) bool { + return validWebSkillName(facts.Skill) && facts.Normalizer == "agent-skill-package/v1" && webSkillSHA256.MatchString(facts.IndexSHA256) && strings.HasPrefix(facts.RawTreeDigest, "sha256:") && webSkillSHA256.MatchString(strings.TrimPrefix(facts.RawTreeDigest, "sha256:")) +} + +func validateWebSkillPinnedFiles(facts WebSkillSourceFacts) error { + index, _ := url.Parse(facts.IndexURL) + seen := make(map[string]bool, len(facts.Files)) + hasSkill := false + for _, file := range facts.Files { + if err := ValidateWebSkillFilePath(file.Path); err != nil || !webSkillSHA256.MatchString(file.SHA256) { + return fmt.Errorf("invalid pinned Skill file") + } + key := file.Path + if seen[key] { + return fmt.Errorf("duplicate pinned Skill file") + } + seen[key] = true + download := *index + download.Path = path.Dir(index.Path) + "/" + facts.Skill + "/" + file.Path + download.RawPath = "" + if file.URL != download.String() { + return fmt.Errorf("Skill file URL differs from its pinned index directory") + } + hasSkill = hasSkill || file.Path == "SKILL.md" + } + if !hasSkill { + return fmt.Errorf("Skill source must pin SKILL.md") + } + return validateWebSkillExactParents(seen) +} + +func validateWebSkillExactParents(seen map[string]bool) error { + for key := range seen { + for parent := path.Dir(key); parent != "."; parent = path.Dir(parent) { + if seen[parent] { + return fmt.Errorf("Skill file paths overlap a directory") + } + } + } + return nil +} + +func CloneWebSkillSourceFacts(facts *WebSkillSourceFacts) *WebSkillSourceFacts { + if facts == nil { + return nil + } + cloned := *facts + cloned.Files = append([]WebSkillFile(nil), facts.Files...) + return &cloned +} diff --git a/install/integrationctl/agentplugins/domain/web_skill_policy_test.go b/install/integrationctl/agentplugins/domain/web_skill_policy_test.go new file mode 100644 index 000000000..b63735a1e --- /dev/null +++ b/install/integrationctl/agentplugins/domain/web_skill_policy_test.go @@ -0,0 +1,37 @@ +package domain + +import "testing" + +func TestWebSkillEndpointPreservesPrefixAndRejectsAmbiguousCoordinates(t *testing.T) { + for input, expected := range map[string]string{"https://Skills.Example/": "https://skills.example", "https://Skills.Example/prefix/": "https://skills.example/prefix"} { + if actual, err := NormalizeWebSkillURL(input); err != nil || actual != expected { + t.Fatalf("endpoint %q: %q %v", input, actual, err) + } + } + for _, input := range []string{"http://skills.example", "https://user@skills.example", "https://skills.example?", "https://skills.example#", "https://skills.example/a/../b", "https://skills.example/a/%2e%2e/b", "https://skills.example/a%2fb", "https://skills.example/a%5cb", "https://skills.example/a//b", "https://skills.example/%0a"} { + if _, err := NormalizeWebSkillURL(input); err == nil { + t.Fatalf("unsafe endpoint accepted: %q", input) + } + } +} + +func TestWebSkillBindingIdentityDependsOnPublisherAndName(t *testing.T) { + first := SourceIdentity{WebSkill: &WebSkillSourceFacts{URL: "https://skills.example/prefix", Skill: "pdf"}, SourceBindingHint: "uap-dependency-web-skill"} + id := ComputeSourceBindingID(first) + second := first + second.WebSkill = CloneWebSkillSourceFacts(first.WebSkill) + second.WebSkill.RawTreeDigest = "new revision" + second.StableSourceKey = "caller override" + if ComputeSourceBindingID(second) != id { + t.Fatal("mutable revision or caller key changed HTTPS Skill identity") + } + second.WebSkill.Skill = "other" + if ComputeSourceBindingID(second) == id { + t.Fatal("different Skills share a binding") + } + second.WebSkill.Skill = "pdf" + second.WebSkill.URL += "/other" + if ComputeSourceBindingID(second) == id { + t.Fatal("different publisher prefixes share a binding") + } +} diff --git a/install/integrationctl/agentplugins/environment/coordinator.go b/install/integrationctl/agentplugins/environment/coordinator.go index 5d119ab3c..1bb3e4fad 100644 --- a/install/integrationctl/agentplugins/environment/coordinator.go +++ b/install/integrationctl/agentplugins/environment/coordinator.go @@ -47,6 +47,7 @@ func (coordinator Coordinator) Resolve(ctx context.Context, requirements Require moving := selected.Git != nil && selected.Git.Selector.Kind != SelectorCommit moving = moving || selected.Registry != nil && selected.Registry.Selector.Kind == SelectorRange moving = moving || selected.Archive != nil && selected.Archive.SHA256 == "" + moving = moving || selected.Web != nil if !moving { return Lock{}, errors.New("exact commit/release/version pins must be changed in the manifest") } @@ -244,6 +245,9 @@ func sortSyncBindings(bindings []PreparedBinding) { } func lockedRevision(pkg LockedPackage) string { + if pkg.Source.Web != nil { + return pkg.Source.Web.RawTreeDigest + } if pkg.Source.Archive != nil { return "sha256:" + pkg.Source.Archive.SHA256 } diff --git a/install/integrationctl/agentplugins/environment/graph.go b/install/integrationctl/agentplugins/environment/graph.go index 072f06eff..19ec60ba8 100644 --- a/install/integrationctl/agentplugins/environment/graph.go +++ b/install/integrationctl/agentplugins/environment/graph.go @@ -112,19 +112,7 @@ func validatePin(pkg LockedPackage) error { if !aliasPattern.MatchString(pkg.Alias) { return fmt.Errorf("invalid locked alias %q", pkg.Alias) } - req := Requirement{Alias: pkg.Alias} - if pkg.Source.Git != nil { - source := pkg.Source.Git - req.Git = &GitSource{Repository: source.Repository, URL: source.URL, Path: source.Path, Selector: source.Requested, Kind: source.Kind, Skill: source.Skill} - } - if pkg.Source.Registry != nil { - source := pkg.Source.Registry - req.Registry = &RegistrySource{Distribution: source.Distribution, Selector: source.Requested} - } - if pkg.Source.Archive != nil { - source := pkg.Source.Archive - req.Archive = &ArchiveSource{URL: source.URL, Path: source.Path, SHA256: source.RequestedSHA256} - } + req := pkg.Requirement() key, err := req.SourceKey() if err != nil { return err @@ -185,6 +173,10 @@ func validateRequiredBySource(pkg LockedPackage, edge RequiredBy) error { if edge.Selector != (Selector{Kind: SelectorArchive, Value: pkg.Source.Archive.RequestedSHA256}) { return fmt.Errorf("%s: archive selector differs from consumer %s", pkg.Alias, edge.Alias) } + } else if pkg.Source.Web != nil { + if edge.Selector != (Selector{Kind: SelectorDefault}) { + return fmt.Errorf("%s: website selector differs from consumer %s", pkg.Alias, edge.Alias) + } } else if edge.Selector.Kind != SelectorRelease && edge.Selector.Kind != SelectorVersion && edge.Selector.Kind != SelectorRange { return fmt.Errorf("%s: invalid registry dependency selector", pkg.Alias) } else if !registrySelectorMatches(edge.Selector, pkg) { diff --git a/install/integrationctl/agentplugins/environment/model.go b/install/integrationctl/agentplugins/environment/model.go index f4986cea2..7adafbff0 100644 --- a/install/integrationctl/agentplugins/environment/model.go +++ b/install/integrationctl/agentplugins/environment/model.go @@ -79,7 +79,16 @@ type LockedArchiveSource struct { SHA256 string `json:"sha256"` } +type WebSource struct { + URL string `json:"url"` + Skill string `json:"skill"` +} + +// LockedWebSource records the complete immutable website acquisition contract. +type LockedWebSource = domain.WebSkillSourceFacts + type Requirement struct { + Web *WebSource `json:"web,omitempty"` Archive *ArchiveSource `json:"archive,omitempty"` Alias string `json:"alias"` Git *GitSource `json:"git,omitempty"` @@ -93,6 +102,7 @@ type Requirements struct { } type LockedSource struct { + Web *LockedWebSource `json:"web,omitempty"` Archive *LockedArchiveSource `json:"archive,omitempty"` Git *LockedGitSource `json:"git,omitempty"` Registry *LockedRegistrySource `json:"registry,omitempty"` @@ -253,8 +263,11 @@ func (requirement Requirement) Validate() error { if !validDeclarationKey(requirement.Alias) { return fmt.Errorf("alias %q must match %s", requirement.Alias, aliasPattern) } - if sourceCount(requirement.Git != nil, requirement.Registry != nil, requirement.Archive != nil) != 1 { - return errors.New("dependency must contain exactly one git, archive or registry source") + if sourceCount(requirement.Git != nil, requirement.Registry != nil, requirement.Archive != nil, requirement.Web != nil) != 1 { + return errors.New("dependency must contain exactly one git, archive, registry or website source") + } + if requirement.Web != nil { + return requirement.Web.Validate() } if requirement.Archive != nil { return requirement.Archive.Validate() @@ -342,6 +355,9 @@ func (requirement Requirement) SourceKey() (string, error) { if err := requirement.Validate(); err != nil { return "", err } + if requirement.Web != nil { + return "https-skill:" + requirement.Web.URL + ":" + requirement.Web.Skill, nil + } if requirement.Git != nil { if requirement.Git.Kind == SkillKind { return "github-skill:" + strings.ToLower(requirement.Git.Repository) + ":" + requirement.Git.Skill + ":" + requirement.Git.Path, nil @@ -394,7 +410,7 @@ func ManifestDigest(requirements Requirements) (string, error) { item.Path = requirement.Archive.Path item.Selector = requirement.RequestedSelector() } else { - item.Selector = requirement.Registry.Selector + item.Selector = requirement.RequestedSelector() } intent.Dependencies = append(intent.Dependencies, item) } @@ -448,8 +464,11 @@ func (lock Lock) ValidateCompatiblePins(_ Requirements) error { } func validateLockedSelector(requirement Requirement, pkg LockedPackage) error { + if requirement.Web != nil { + return validateLockedWeb(requirement, pkg) + } if requirement.Git != nil { - if pkg.Source.Git == nil || pkg.Source.Registry != nil || pkg.Source.Archive != nil || pkg.Source.Git.URL != requirement.Git.URL || !strings.EqualFold(pkg.Source.Git.Repository, requirement.Git.Repository) || + if pkg.Source.Git == nil || pkg.Source.Registry != nil || pkg.Source.Archive != nil || pkg.Source.Web != nil || pkg.Source.Git.URL != requirement.Git.URL || !strings.EqualFold(pkg.Source.Git.Repository, requirement.Git.Repository) || pkg.Source.Git.Path != requirement.Git.Path || pkg.Source.Git.Kind != requirement.Git.Kind || pkg.Source.Git.Skill != requirement.Git.Skill || pkg.Source.Git.Requested != requirement.Git.Selector || !commitPattern.MatchString(pkg.Source.Git.Commit) { return errors.New("locked Git identity does not match manifest") @@ -470,7 +489,7 @@ func validateLockedSelector(requirement Requirement, pkg LockedPackage) error { func validateLockedRegistrySelector(requirement Requirement, pkg LockedPackage) error { source := pkg.Source.Registry - if source == nil || pkg.Source.Git != nil || pkg.Source.Archive != nil || !strings.EqualFold(source.Distribution, requirement.Registry.Distribution) || + if source == nil || pkg.Source.Git != nil || pkg.Source.Archive != nil || pkg.Source.Web != nil || !strings.EqualFold(source.Distribution, requirement.Registry.Distribution) || source.Requested != requirement.Registry.Selector || source.ReleaseSequence == 0 || !registrySelectorMatches(requirement.Registry.Selector, pkg) || !strings.EqualFold(source.DistributionID, source.Distribution) || !distributionPattern.MatchString(source.DistributionID) || !repositoryPattern.MatchString(source.Repository) || validateSubpath(source.Path) != nil || !commitPattern.MatchString(source.Commit) { return errors.New("locked registry identity does not match manifest") diff --git a/install/integrationctl/agentplugins/environment/resolve_graph.go b/install/integrationctl/agentplugins/environment/resolve_graph.go index 3941e41ff..47ca10914 100644 --- a/install/integrationctl/agentplugins/environment/resolve_graph.go +++ b/install/integrationctl/agentplugins/environment/resolve_graph.go @@ -193,7 +193,7 @@ func (solver *graphSolver) choose(ctx context.Context, key string, node graphNod } } } - if node.requirement.Git != nil || node.requirement.Archive != nil || node.requirement.Registry.Selector.Kind == SelectorRelease { + if node.requirement.Git != nil || node.requirement.Archive != nil || node.requirement.Web != nil || node.requirement.Registry.Selector.Kind == SelectorRelease { return solver.chooseExact(ctx, key, node, choices, last) } return solver.chooseCandidates(ctx, key, node, choices, last) diff --git a/install/integrationctl/agentplugins/environment/skills.go b/install/integrationctl/agentplugins/environment/skills.go index 66f0657b6..aae8d6226 100644 --- a/install/integrationctl/agentplugins/environment/skills.go +++ b/install/integrationctl/agentplugins/environment/skills.go @@ -20,7 +20,10 @@ func validateSkillSource(source *GitSource) error { if source.URL != "" || !repositoryPattern.MatchString(source.Repository) { return errors.New("standalone Skills require a GitHub owner/repo source") } - name := source.Skill + return validateSkillName(source.Skill) +} + +func validateSkillName(name string) error { if n := utf8.RuneCountInString(name); n < 1 || n > 64 || strings.HasPrefix(name, "-") || strings.HasSuffix(name, "-") || strings.Contains(name, "--") { return errors.New("invalid expected Skill name") } diff --git a/install/integrationctl/agentplugins/environment/sources.go b/install/integrationctl/agentplugins/environment/sources.go index 0d12b5650..57b53b792 100644 --- a/install/integrationctl/agentplugins/environment/sources.go +++ b/install/integrationctl/agentplugins/environment/sources.go @@ -58,6 +58,9 @@ func (source ArchiveSource) Validate() error { } func (requirement Requirement) RequestedSelector() Selector { + if requirement.Web != nil { + return Selector{Kind: SelectorDefault} + } if requirement.Git != nil { return requirement.Git.Selector } @@ -73,7 +76,7 @@ func (requirement Requirement) RequestedSelector() Selector { func validateLockedArchive(requirement Requirement, pkg LockedPackage) error { a := pkg.Source.Archive r := requirement.Archive - if a == nil || pkg.Source.Git != nil || pkg.Source.Registry != nil || a.URL != r.URL || a.Path != r.Path || a.RequestedSHA256 != r.SHA256 || !sourcespec.ValidSHA256(a.SHA256) || (r.SHA256 != "" && r.SHA256 != a.SHA256) { + if a == nil || pkg.Source.Git != nil || pkg.Source.Registry != nil || pkg.Source.Web != nil || a.URL != r.URL || a.Path != r.Path || a.RequestedSHA256 != r.SHA256 || !sourcespec.ValidSHA256(a.SHA256) || (r.SHA256 != "" && r.SHA256 != a.SHA256) { return errors.New("locked archive identity does not match manifest") } return nil @@ -83,6 +86,9 @@ func validateLockedArchive(requirement Requirement, pkg LockedPackage) error { // adopts mutable store provenance or a machine's profile/project paths. func (pkg LockedPackage) Requirement() Requirement { r := Requirement{Alias: pkg.Alias} + if s := pkg.Source.Web; s != nil { + r.Web = &WebSource{URL: s.URL, Skill: s.Skill} + } if s := pkg.Source.Git; s != nil { r.Git = &GitSource{Repository: s.Repository, URL: s.URL, Path: s.Path, Selector: s.Requested, Kind: s.Kind, Skill: s.Skill} } diff --git a/install/integrationctl/agentplugins/environment/web.go b/install/integrationctl/agentplugins/environment/web.go new file mode 100644 index 000000000..3e11d3501 --- /dev/null +++ b/install/integrationctl/agentplugins/environment/web.go @@ -0,0 +1,26 @@ +package environment + +import ( + "errors" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" +) + +// NormalizeWebURL canonicalizes the publisher endpoint used by portable intent. +func NormalizeWebURL(endpoint string) (string, error) { return domain.NormalizeWebSkillURL(endpoint) } + +func (source WebSource) Validate() error { + endpoint, err := NormalizeWebURL(source.URL) + if err != nil || endpoint != source.URL { + return errors.New("website Skill source requires a normalized credential-free HTTPS URL") + } + return validateSkillName(source.Skill) +} + +func validateLockedWeb(requirement Requirement, pkg LockedPackage) error { + source := pkg.Source.Web + if source == nil || pkg.Source.Git != nil || pkg.Source.Registry != nil || pkg.Source.Archive != nil || source.URL != requirement.Web.URL || source.Skill != requirement.Web.Skill { + return errors.New("locked website Skill identity does not match manifest") + } + return domain.ValidateWebSkillSourceFacts(*source) +} diff --git a/install/integrationctl/agentplugins/environment/web_test.go b/install/integrationctl/agentplugins/environment/web_test.go new file mode 100644 index 000000000..d2421c2d7 --- /dev/null +++ b/install/integrationctl/agentplugins/environment/web_test.go @@ -0,0 +1,94 @@ +package environment + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" +) + +// The pin contract must reject source mixing and any loss of acquisition facts +// before a locked Skill can be restored or assigned to an environment. +func TestWebsiteSkillPinRoundTripAndValidation(t *testing.T) { + req := Requirement{Alias: "pdf", Web: &WebSource{URL: "https://publisher.example/prefix", Skill: "pdf"}} + key, err := req.SourceKey() + if err != nil || key != "https-skill:https://publisher.example/prefix:pdf" { + t.Fatalf("%s: %v", key, err) + } + pin := LockedPackage{Alias: "pdf", SourceKey: key, DigestAlgorithm: TreeDigestAlgorithm, TreeDigest: "sha256:" + strings.Repeat("a", 64), ManifestDigest: "sha256:" + strings.Repeat("b", 64), Targets: []string{"codex"}, Source: LockedSource{Web: &LockedWebSource{URL: req.Web.URL, Skill: "pdf", IndexURL: req.Web.URL + "/.well-known/skills/index.json", IndexSHA256: strings.Repeat("c", 64), Files: []domain.WebSkillFile{{Path: "SKILL.md", URL: req.Web.URL + "/.well-known/skills/pdf/SKILL.md", SHA256: strings.Repeat("d", 64)}}, RawTreeDigest: "sha256:" + strings.Repeat("e", 64), Normalizer: SkillNormalizerV1}}} + reqs := Requirements{SchemaVersion: 1, Dependencies: map[string]Requirement{"pdf": req}, Targets: map[string][]string{"codex": {"pdf"}}} + digest, err := ManifestDigest(reqs) + if err != nil { + t.Fatal(err) + } + lock := Lock{LockVersion: LockVersion, ManifestDigest: digest, Packages: []LockedPackage{pin}} + body, err := json.Marshal(lock) + if err != nil { + t.Fatal(err) + } + var decoded Lock + if err := json.Unmarshal(body, &decoded); err != nil { + t.Fatal(err) + } + if err := decoded.Validate(reqs); err != nil { + t.Fatal(err) + } + coordinator := Coordinator{Resolver: websitePinResolver{pin: pin}} + resolved, err := coordinator.Resolve(t.Context(), reqs, Lock{}, "") + if err != nil { + t.Fatal(err) + } + if _, err := coordinator.Resolve(t.Context(), reqs, resolved, "pdf"); err != nil { + t.Fatal(err) + } + plan, err := coordinator.PrepareSync(reqs, resolved, []BindingObservation{{Alias: "pdf", ClientID: "codex", SourceKey: key, PackageDigest: pin.TreeDigest, Revision: pin.Source.Web.RawTreeDigest, Managed: true}}) + if err != nil || len(plan.Bindings) != 1 || plan.Bindings[0].Action != ActionUnchanged { + t.Fatalf("website sync changed stable pin: %+v, %v", plan, err) + } + if decoded.Packages[0].Revision() != pin.Source.Web.RawTreeDigest || decoded.Packages[0].Requirement().RequestedSelector() != (Selector{Kind: SelectorDefault}) { + t.Fatal("lost website source identity") + } + reconstructed := decoded.Packages[0].Requirement() + reconstructed.Web.Skill = "other" + if decoded.Packages[0].Source.Web.Skill != "pdf" { + t.Fatal("requirement aliases locked source") + } + for _, mutate := range []func(*LockedPackage){ + func(p *LockedPackage) { p.Source.Web = nil }, + func(p *LockedPackage) { + p.Source.Git = &LockedGitSource{Repository: "acme/tools", Requested: Selector{Kind: SelectorDefault}} + }, + func(p *LockedPackage) { p.Source.Web.Skill = "other" }, + func(p *LockedPackage) { p.Source.Web.IndexSHA256 = "" }, + func(p *LockedPackage) { p.Source.Web.RawTreeDigest = "" }, + func(p *LockedPackage) { p.Source.Web.Normalizer = "future" }, + func(p *LockedPackage) { p.Source.Web.Files = nil }, + func(p *LockedPackage) { p.Source.Web.Files[0].URL = "https://elsewhere.example/SKILL.md" }, + func(p *LockedPackage) { p.Source.Web.Files[0].Path = "../SKILL.md" }, + } { + var bad LockedPackage + body, _ := json.Marshal(pin) + if err := json.Unmarshal(body, &bad); err != nil { + t.Fatal(err) + } + mutate(&bad) + if err := validatePin(bad); err == nil { + t.Fatalf("accepted malformed website lock: %+v", bad) + } + } +} + +// Embed the existing graph fixture so accidental Directory enumeration fails. +type websitePinResolver struct { + remoteGraphResolver + pin LockedPackage +} + +func (resolver websitePinResolver) Resolve(_ context.Context, req Requirement, targets []string, _ *LockedPackage, _ bool) (LockedPackage, error) { + pin := resolver.pin + pin.Alias = req.Alias + pin.Targets = append([]string(nil), targets...) + return pin, nil +} diff --git a/install/integrationctl/agentplugins/installer/acquired_skill.go b/install/integrationctl/agentplugins/installer/acquired_skill.go index e71d6d1cc..a164e2d21 100644 --- a/install/integrationctl/agentplugins/installer/acquired_skill.go +++ b/install/integrationctl/agentplugins/installer/acquired_skill.go @@ -14,7 +14,7 @@ import ( // contract, never a replacement for a direct transport's derived source key. func acquiredSkillSourceKey(source domain.SourceIdentity, portable bool) (string, error) { facts := source.Skill - if !portable || facts == nil || source.SourceBindingHint != "uap-dependency-skill" || source.SourceKind != "" || source.SourceURL != "" || source.ArchiveSHA256 != "" { + if !portable || facts == nil || source.WebSkill != nil || source.SourceBindingHint != "uap-dependency-skill" || source.SourceKind != "" || source.SourceURL != "" || source.ArchiveSHA256 != "" { return "", ErrInvalidRequest } _, repository, err := sourcespec.NormalizeGitURL("https://github.com/" + source.Repository + ".git") @@ -24,16 +24,32 @@ func acquiredSkillSourceKey(source domain.SourceIdentity, portable bool) (string if facts.Normalizer != "agent-skill-package/v1" || !strings.HasPrefix(facts.RawTreeDigest, "sha256:") || !sourcespec.ValidSHA256(strings.TrimPrefix(facts.RawTreeDigest, "sha256:")) { return "", ErrInvalidRequest } + if !validAcquiredSkillPaths(*facts, source.PackageSubpath) { + return "", ErrInvalidRequest + } + key := "github-skill:" + strings.ToLower(repository) + ":" + facts.Name + ":" + facts.RequestedPath + if source.StableSourceKey != key { + return "", ErrInvalidRequest + } + return key, nil +} + +func validAcquiredSkillPaths(facts domain.SkillSourceFacts, packageSubpath string) bool { for _, value := range []string{facts.RequestedPath, facts.ResolvedPath} { if normalized, err := sourcespec.NormalizeSubpath(value); err != nil || normalized != value { - return "", ErrInvalidRequest + return false } } - if facts.ResolvedPath != source.PackageSubpath || (facts.ResolvedPath != "" && path.Base(facts.ResolvedPath) != facts.Name) { + return facts.ResolvedPath == packageSubpath && (facts.ResolvedPath == "" || path.Base(facts.ResolvedPath) == facts.Name) +} + +func acquiredWebSkillSourceKey(source domain.SourceIdentity, portable bool) (string, error) { + facts := source.WebSkill + if !portable || facts == nil || source.Skill != nil || source.SourceBindingHint != "uap-dependency-web-skill" || source.SourceKind != "https-skill" || source.ArchiveSHA256 != "" || source.Repository != "" || source.PackageSubpath != "" || domain.ValidateWebSkillSourceFacts(*facts) != nil { return "", ErrInvalidRequest } - key := "github-skill:" + strings.ToLower(repository) + ":" + facts.Name + ":" + facts.RequestedPath - if source.StableSourceKey != key { + key := "https-skill:" + facts.URL + ":" + facts.Skill + if source.SourceURL != facts.URL || source.StableSourceKey != key || source.CanonicalSource != key || source.RequestedSource != key || source.ResolvedRevision != facts.RawTreeDigest { return "", ErrInvalidRequest } return key, nil diff --git a/install/integrationctl/agentplugins/installer/acquired_source.go b/install/integrationctl/agentplugins/installer/acquired_source.go index efca3bb7b..cdbfd5bea 100644 --- a/install/integrationctl/agentplugins/installer/acquired_source.go +++ b/install/integrationctl/agentplugins/installer/acquired_source.go @@ -44,6 +44,12 @@ func acquiredDirectSourceKey(source domain.SourceIdentity, portable bool) (strin func validateAcquiredSource(acquired *domain.AcquiredSnapshot, portable bool) (string, error) { source := acquired.Source + if source.SourceKind == "https-skill" || source.WebSkill != nil || source.SourceBindingHint == "uap-dependency-web-skill" { + if acquired.OriginMode != domain.OriginModeDirect { + return "", ErrInvalidRequest + } + return acquiredWebSkillSourceKey(source, portable) + } if source.SourceKind != "" { if acquired.OriginMode != domain.OriginModeDirect || source.Skill != nil || source.StableSourceKey != "" || source.SourceBindingHint == "uap-dependency-skill" { return "", ErrInvalidRequest diff --git a/install/integrationctl/agentplugins/installer/acquired_web_skill_test.go b/install/integrationctl/agentplugins/installer/acquired_web_skill_test.go new file mode 100644 index 000000000..1c663c702 --- /dev/null +++ b/install/integrationctl/agentplugins/installer/acquired_web_skill_test.go @@ -0,0 +1,55 @@ +package installer + +import ( + "strings" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" +) + +func acquiredWebSkillFixture() domain.AcquiredSnapshot { + key := "https-skill:https://skills.example/prefix:pdf" + facts := &domain.WebSkillSourceFacts{URL: "https://skills.example/prefix", Skill: "pdf", IndexURL: "https://skills.example/prefix/.well-known/agent-skills/index.json", IndexSHA256: strings.Repeat("a", 64), RawTreeDigest: "sha256:" + strings.Repeat("b", 64), Normalizer: "agent-skill-package/v1", Files: []domain.WebSkillFile{{Path: "SKILL.md", URL: "https://skills.example/prefix/.well-known/agent-skills/pdf/SKILL.md", SHA256: strings.Repeat("c", 64)}}} + return domain.AcquiredSnapshot{OriginMode: domain.OriginModeDirect, Source: domain.SourceIdentity{SourceKind: "https-skill", SourceURL: facts.URL, WebSkill: facts, StableSourceKey: key, RequestedSource: key, CanonicalSource: key, SourceBindingHint: "uap-dependency-web-skill", ResolvedRevision: facts.RawTreeDigest}} +} + +func TestAcquiredWebSkillRequiresExactPinnedAuthority(t *testing.T) { + valid := acquiredWebSkillFixture() + if key, err := validateAcquiredSource(&valid, true); err != nil || key != valid.Source.StableSourceKey { + t.Fatalf("valid HTTPS Skill refused: %q %v", key, err) + } + for name, mutate := range map[string]func(*domain.SourceIdentity){ + "override": func(s *domain.SourceIdentity) { s.StableSourceKey = "https-skill:https://other.example:pdf" }, + "coordinate": func(s *domain.SourceIdentity) { s.SourceURL = "https://other.example" }, + "revision": func(s *domain.SourceIdentity) { s.ResolvedRevision = "sha256:" + strings.Repeat("d", 64) }, + "git identity": func(s *domain.SourceIdentity) { s.Repository = "acme/tools" }, + "missing facts": func(s *domain.SourceIdentity) { s.WebSkill = nil }, + "normalizer": func(s *domain.SourceIdentity) { s.WebSkill.Normalizer = "future" }, + "name": func(s *domain.SourceIdentity) { s.WebSkill.Skill = "../pdf" }, + "raw digest": func(s *domain.SourceIdentity) { s.WebSkill.RawTreeDigest = "sha256:invalid" }, + "index checksum": func(s *domain.SourceIdentity) { s.WebSkill.IndexSHA256 = strings.Repeat("A", 64) }, + "file checksum": func(s *domain.SourceIdentity) { s.WebSkill.Files[0].SHA256 = "invalid" }, + "index origin": func(s *domain.SourceIdentity) { + s.WebSkill.IndexURL = "https://other.example/.well-known/agent-skills/index.json" + }, + "file origin": func(s *domain.SourceIdentity) { s.WebSkill.Files[0].URL = "https://other.example/SKILL.md" }, + "file traversal": func(s *domain.SourceIdentity) { s.WebSkill.Files[0].Path = "../SKILL.md" }, + "file collision": func(s *domain.SourceIdentity) { s.WebSkill.Files = append(s.WebSkill.Files, s.WebSkill.Files[0]) }, + "missing entrypoint": func(s *domain.SourceIdentity) { s.WebSkill.Files = nil }, + } { + t.Run(name, func(t *testing.T) { + bad := acquiredWebSkillFixture() + mutate(&bad.Source) + if _, err := validateAcquiredSource(&bad, true); err == nil { + t.Fatal("tampered HTTPS Skill authority admitted") + } + }) + } + if _, err := validateAcquiredSource(&valid, false); err == nil { + t.Fatal("HTTPS Skill accepted without portable environment authority") + } + valid.OriginMode = domain.OriginModeDirectory + if _, err := validateAcquiredSource(&valid, true); err == nil { + t.Fatal("HTTPS Skill accepted as Directory release") + } +} diff --git a/install/integrationctl/agentplugins/installer/prepare.go b/install/integrationctl/agentplugins/installer/prepare.go index 0b57e26e0..048248d2c 100644 --- a/install/integrationctl/agentplugins/installer/prepare.go +++ b/install/integrationctl/agentplugins/installer/prepare.go @@ -418,6 +418,9 @@ func (e *Engine) loadMutatingPackage(ctx context.Context, handle *PreparedOperat if err != nil { return err } + if snapshot.Source.WebSkill != nil && (envelope.LoaderKind != domain.LoaderKindAgentPlugins || envelope.FormatID != domain.FormatIDAgentPluginsV1 || len(envelope.Skills) != 1 || envelope.Skills[snapshot.Source.WebSkill.Skill].Name != snapshot.Source.WebSkill.Skill) { + return fmt.Errorf("%w: HTTPS Skill acquisition requires its normalized portable Skill package", ErrInvalidRequest) + } handle.envelope = envelope return nil } diff --git a/install/integrationctl/agentplugins/usecase/installation.go b/install/integrationctl/agentplugins/usecase/installation.go index bab136433..5657f3f3b 100644 --- a/install/integrationctl/agentplugins/usecase/installation.go +++ b/install/integrationctl/agentplugins/usecase/installation.go @@ -368,7 +368,8 @@ func managedDirectSource(installation domain.Installation, input AddInput) bool return false } if installation.Source.StableSourceKey != "" { - return input.Envelope.Source.SourceBindingHint == "uap-dependency-skill" && installation.Source.StableSourceKey == input.Envelope.Source.StableSourceKey && input.EnvironmentAuthority.SourceKey == installation.Source.StableSourceKey + hint := input.Envelope.Source.SourceBindingHint + return (hint == "uap-dependency-skill" || hint == "uap-dependency-web-skill") && installation.Source.StableSourceKey == input.Envelope.Source.StableSourceKey && input.EnvironmentAuthority.SourceKey == installation.Source.StableSourceKey } return installation.Source.PackageSubpath == input.Envelope.Source.PackageSubpath } diff --git a/install/integrationctl/agentplugins/usecase/service.go b/install/integrationctl/agentplugins/usecase/service.go index 27bdcef7f..30caaaf8c 100644 --- a/install/integrationctl/agentplugins/usecase/service.go +++ b/install/integrationctl/agentplugins/usecase/service.go @@ -387,8 +387,9 @@ func immutableDirectGit(source domain.SourceBinding) bool { func packageRevisionFromEnvelope(envelope domain.PackageEnvelope) *domain.ClientPackageRevision { return &domain.ClientPackageRevision{ - Skill: domain.CloneSkillSourceFacts(envelope.Source.Skill), - Version: envelope.Manifest.Version, ResolvedRevision: envelope.Source.ResolvedRevision, + Skill: domain.CloneSkillSourceFacts(envelope.Source.Skill), + WebSkill: domain.CloneWebSkillSourceFacts(envelope.Source.WebSkill), + Version: envelope.Manifest.Version, ResolvedRevision: envelope.Source.ResolvedRevision, TreeDigest: envelope.TreeDigest, ManifestDigest: envelope.ManifestDigest, CatalogEvidence: cloneCatalogEvidence(envelope.CatalogEvidence), } @@ -405,6 +406,7 @@ func packageRevisionForInput(input AddInput) *domain.ClientPackageRevision { func packageRevisionMatches(revision *domain.ClientPackageRevision, envelope domain.PackageEnvelope) bool { return revision != nil && revision.TreeDigest == envelope.TreeDigest && revision.ManifestDigest == envelope.ManifestDigest && + reflect.DeepEqual(revision.WebSkill, envelope.Source.WebSkill) && reflect.DeepEqual(revision.CatalogEvidence, envelope.CatalogEvidence) } diff --git a/install/integrationctl/agentplugins/usecase/web_skill_revision_test.go b/install/integrationctl/agentplugins/usecase/web_skill_revision_test.go new file mode 100644 index 000000000..c99ae67b3 --- /dev/null +++ b/install/integrationctl/agentplugins/usecase/web_skill_revision_test.go @@ -0,0 +1,24 @@ +package usecase + +import ( + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" +) + +func TestWebSkillRevisionFreezesFilesForRepair(t *testing.T) { + facts := &domain.WebSkillSourceFacts{URL: "https://skills.example", Skill: "pdf", RawTreeDigest: "old revision", Files: []domain.WebSkillFile{{Path: "SKILL.md", SHA256: "old hash"}}} + envelope := domain.PackageEnvelope{Source: domain.SourceIdentity{WebSkill: facts}, TreeDigest: "normalized", ManifestDigest: "manifest"} + revision := packageRevisionFromEnvelope(envelope) + if !repairPackageRevisionMatches(revision, envelope, false) { + t.Fatal("captured revision cannot be repaired") + } + facts.Files[0].SHA256 = "new hash" + facts.RawTreeDigest = "new revision" + if revision.WebSkill.Files[0].SHA256 != "old hash" || revision.WebSkill.RawTreeDigest != "old revision" { + t.Fatal("caller mutation rewrote an applied client's repair facts") + } + if repairPackageRevisionMatches(revision, envelope, false) { + t.Fatal("repair accepted different pinned file provenance") + } +} From 19f12263ce9a87d327fa0d5038cb81007adf9604 Mon Sep 17 00:00:00 2001 From: iliya Date: Sat, 24 Oct 2026 16:33:34 +0300 Subject: [PATCH 22/25] fix(sources): preserve literal plus in archive selectors --- .../source_archive_plus_e2e_test.go | 41 +++++++++++++++++++ .../review-corrections-restoration-plan.md | 3 +- .../archive_fragment_regression_test.go | 12 ++++++ .../agentplugins/sourcespec/parse.go | 3 +- 4 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 cli/internal/agentpluginscli/source_archive_plus_e2e_test.go diff --git a/cli/internal/agentpluginscli/source_archive_plus_e2e_test.go b/cli/internal/agentpluginscli/source_archive_plus_e2e_test.go new file mode 100644 index 000000000..da56d1108 --- /dev/null +++ b/cli/internal/agentpluginscli/source_archive_plus_e2e_test.go @@ -0,0 +1,41 @@ +package agentpluginscli + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" +) + +// Literal fragment plus signs must select the same complete package on first +// install and a cold locked restore, including supporting file bytes. +func TestArchiveLiteralPlusSelectorSavedAndColdRestore(t *testing.T) { + transport := newDirectTransportFixture(t, "zip", false) + root := t.TempDir() + packagePath := filepath.Join(root, "plugins", "c++") + aliasGraphPackage(t, packagePath, "plus-package") + dependencyWrite(t, filepath.Join(packagePath, "reference.txt"), "selected c++ package resource") + transport.payload = directArchiveBytes(t, root, "zip", "publisher-wrapper/") + checksum := sha256.Sum256(transport.payload) + source := transport.server.URL + "/package.zip#sha256=" + hex.EncodeToString(checksum[:]) + "&path=plugins/c++" + fixture := directCLIEnvironment(t, transport) + dependencyRun(t, fixture, "add", source, "--target", "gemini", "--format", "json") + pin := directLockedPin(t, fixture) + if pin.Source.Archive == nil || pin.Source.Archive.Path != "plugins/c++" { + t.Fatal("literal plus path lost", pin) + } + state, err := fixture.store.Load() + if err != nil { + t.Fatal(err) + } + for _, binding := range state.Installations[0].Clients { + if got := dependencyRead(t, filepath.Join(binding.TargetLocator, "reference.txt")); got != "selected c++ package resource" { + t.Fatal("wrong package projected", got) + } + } + if err := os.RemoveAll(fixture.app.EnvironmentStoreRoot); err != nil { + t.Fatal(err) + } + directColdRestore(t, fixture, transport, pin) +} diff --git a/docs/plans/source-transports-e2e/review-corrections-restoration-plan.md b/docs/plans/source-transports-e2e/review-corrections-restoration-plan.md index 3add41f43..d8dc7817d 100644 --- a/docs/plans/source-transports-e2e/review-corrections-restoration-plan.md +++ b/docs/plans/source-transports-e2e/review-corrections-restoration-plan.md @@ -11,7 +11,8 @@ two independent final code reviews before root publishes the exact checkpoint. ## 1. Decode archive selectors exactly once URL parsing already decodes Fragment. Feed u.EscapedFragment() to the existing -url.ParseQuery decoder and clear both Fragment and RawFragment on the cloned +percent decoder with literal plus preserved (replace raw + with %2B before +url.ParseQuery), and clear both Fragment and RawFragment on the cloned endpoint before normalization. Local selector grammar is already encoded input and remains unchanged. Test HTTPS/local parity for literal plus (c++), ampersand, literal percent/%41, slash, Unicode and ordinary selected paths. Preserve malformed diff --git a/install/integrationctl/agentplugins/sourcespec/archive_fragment_regression_test.go b/install/integrationctl/agentplugins/sourcespec/archive_fragment_regression_test.go index dde554f75..c35a54888 100644 --- a/install/integrationctl/agentplugins/sourcespec/archive_fragment_regression_test.go +++ b/install/integrationctl/agentplugins/sourcespec/archive_fragment_regression_test.go @@ -28,3 +28,15 @@ func TestArchiveHTTPSAndLocalSelectorsDecodeExactlyOnce(t *testing.T) { } } } + +func TestArchiveSelectorsPreserveLiteralPlus(t *testing.T) { + checksum := strings.Repeat("b", 64) + for _, endpoint := range []string{"https://packages.test/plugin.zip", "/tmp/plugin.zip"} { + for _, spelling := range []string{"plugins/c++", "plugins/c%2B%2B"} { + spec, err := Parse(endpoint + "#sha256=" + checksum + "&path=" + spelling) + if err != nil || spec.Archive == nil || spec.Archive.Path != "plugins/c++" { + t.Fatalf("literal plus selector changed: %+v %v", spec, err) + } + } + } +} diff --git a/install/integrationctl/agentplugins/sourcespec/parse.go b/install/integrationctl/agentplugins/sourcespec/parse.go index f9e27a640..30616969b 100644 --- a/install/integrationctl/agentplugins/sourcespec/parse.go +++ b/install/integrationctl/agentplugins/sourcespec/parse.go @@ -107,7 +107,8 @@ func parseArchive(u *url.URL) (Archive, error) { func parseArchiveFragment(a Archive, fragment string) (Archive, error) { if fragment != "" { - values, err := url.ParseQuery(fragment) + // Fragments are not form data: preserve literal plus while decoding percent escapes once. + values, err := url.ParseQuery(strings.ReplaceAll(fragment, "+", "%2B")) if err != nil || len(values["sha256"]) != 1 || !ValidSHA256(values.Get("sha256")) { return Archive{}, errors.New("archive fragment requires sha256=FULL_SHA256 and optional path") } From 4c3667ed323d358e3fcd5e674e9e4c6eb0c677d4 Mon Sep 17 00:00:00 2001 From: iliya Date: Sat, 24 Oct 2026 19:09:32 +0300 Subject: [PATCH 23/25] fix(sources): complete website Skill lifecycle checks - Let saved update replace a website Skill pin without requiring the previous pinned bytes; the candidate is still verified before publication. - Report context cancellation during a Skill file body read as context.Canceled, matching the request path. - Cover locked replay refusal after publisher change, deleted pinned bytes, untrusted TLS certificates and the 256-file pin bound. --- .../agentpluginscli/saved_lifecycle.go | 4 +- .../skills_website_e2e_test.go | 37 ++++++++++ .../website-skills-evidence.md | 42 +++++++++++ .../adapters/sourceacquisition/skill_web.go | 3 + .../sourceacquisition/skill_web_test.go | 73 +++++++++++++++++++ .../domain/web_skill_policy_test.go | 26 ++++++- 6 files changed, 183 insertions(+), 2 deletions(-) diff --git a/cli/internal/agentpluginscli/saved_lifecycle.go b/cli/internal/agentpluginscli/saved_lifecycle.go index 20e7f5765..7ab3b7b55 100644 --- a/cli/internal/agentpluginscli/saved_lifecycle.go +++ b/cli/internal/agentpluginscli/saved_lifecycle.go @@ -171,7 +171,9 @@ func runSavedLifecycle(cmd *cobra.Command, app App, opts *options, selector, ope if operation == "remove" && opts.purgeData { return true, errors.New("remove owned bindings before purging retained data; PLUGIN_DATA is preserved") } - snapshot, err := snapshotEnvironment(cmd, app, operation != "remove") + // Update replaces the recorded pins, so their old bytes are not a precondition. + // runSavedEnvironment still verifies every pin of the candidate before publication. + snapshot, err := snapshotEnvironment(cmd, app, operation != "remove" && operation != "update") if err != nil { return true, err } diff --git a/cli/internal/agentpluginscli/skills_website_e2e_test.go b/cli/internal/agentpluginscli/skills_website_e2e_test.go index 790ba3250..aee464541 100644 --- a/cli/internal/agentpluginscli/skills_website_e2e_test.go +++ b/cli/internal/agentpluginscli/skills_website_e2e_test.go @@ -22,6 +22,7 @@ func TestWebsiteSkillsSavedLifecycleAndColdRefusal(t *testing.T) { var mu sync.Mutex version := 1 indexAvailable := true + resourceAvailable := true server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { mu.Lock() defer mu.Unlock() @@ -38,6 +39,10 @@ func TestWebsiteSkillsSavedLifecycleAndColdRefusal(t *testing.T) { case "/scoped/.well-known/agent-skills/pdf/SKILL.md": _, _ = fmt.Fprint(w, "---\nname: pdf\ndescription: Read PDF documents\n---\nRead references/note.txt.\n") case "/scoped/.well-known/agent-skills/pdf/references/note.txt": + if !resourceAvailable { + http.NotFound(w, r) + return + } _, _ = fmt.Fprintf(w, "resource version %d", version) default: http.NotFound(w, r) @@ -103,6 +108,20 @@ func TestWebsiteSkillsSavedLifecycleAndColdRefusal(t *testing.T) { indexAvailable = true version = 2 mu.Unlock() + // With the local store gone, locked replay must fetch the recorded files, + // refuse the changed publisher bytes and leave the lock, state and install intact. + if err := os.RemoveAll(j.cli.app.EnvironmentStoreRoot); err != nil { + t.Fatal(err) + } + installedResource := filepath.Join(addedBinding.TargetLocator, "skills", "pdf", "references", "note.txt") + _, replayLockPath := j.cli.app.dependencyPaths() + replayState, replayLock := dependencyRead(t, j.cli.store.Path), dependencyRead(t, replayLockPath) + if _, _, err := j.cli.execute(false, "deps", "sync", "--locked", "--yes", "--format", "json"); err == nil || !strings.Contains(strings.ToLower(err.Error()), "checksum") { + t.Fatalf("locked replay accepted changed publisher bytes: %v", err) + } + if dependencyRead(t, j.cli.store.Path) != replayState || dependencyRead(t, replayLockPath) != replayLock || dependencyRead(t, installedResource) != "resource version 1" { + t.Fatal("refused locked replay changed state, lock or installed resource") + } j.run(t, "update", "pdf", "--format", "json") _, updated := j.intent(t) if updated.Packages[0].TreeDigest == pin.TreeDigest || updated.Packages[0].SourceKey != pin.SourceKey { @@ -135,6 +154,24 @@ func TestWebsiteSkillsSavedLifecycleAndColdRefusal(t *testing.T) { if len(refused.state(t).Installations) != 0 { t.Fatal("failed import changed profiles") } + // A publisher that deleted the pinned resource must fail the same way, with no + // published profile or lock. + mu.Lock() + resourceAvailable = false + mu.Unlock() + missing := makeJourney() + missingRequirementsPath, missingLockPath := missing.cli.app.dependencyPaths() + if _, _, err = missing.cli.execute(false, "deps", "import", exchange, "--install", "--yes", "--format", "json"); err == nil || !strings.Contains(err.Error(), "404") { + t.Fatalf("deleted historical bytes accepted for cold import: %v", err) + } + for _, p := range []string{missingRequirementsPath, missingLockPath} { + if _, e := os.Stat(p); !os.IsNotExist(e) { + t.Fatalf("failed import published %s: %v", p, e) + } + } + if len(missing.state(t).Installations) != 0 { + t.Fatal("failed import changed profiles") + } } // A staged artifact must not satisfy another pin whose file provenance differs, diff --git a/docs/plans/source-transports-e2e/website-skills-evidence.md b/docs/plans/source-transports-e2e/website-skills-evidence.md index 753723d5d..db45ec2a4 100644 --- a/docs/plans/source-transports-e2e/website-skills-evidence.md +++ b/docs/plans/source-transports-e2e/website-skills-evidence.md @@ -74,3 +74,45 @@ Existing PR #473 and its commits remain the delivery vehicle. This extension does not merge the PR, publish a CLI package, change the landing command to an unsupported published `latest`, or claim every catalog entry is compatible. Discovery artifact format 0.2 and redirects remain explicitly unsupported. + +## Completion pass + +2026-10-24, on PR #473 head `19f12263` before this pass. Source-only and +unreleased; no merge, installer publication or release was performed. + +Defects found by the lifecycle checks and fixed: + +- `update` of a website Skill verified the previous pin's bytes first. With an + empty local store and changed publisher bytes it failed with `skill file + original-byte SHA256 does not match its locked checksum`. Update no longer + requires those old bytes; `runSavedEnvironment` still verifies the whole + candidate, including unchanged pins, before publication. This applies to every + saved update, not only website Skills. +- Cancellation while reading a Skill file body returned `skill download was + incomplete` instead of `context.Canceled`. Body-read cancellation now reports + the context error, as the request path already did. + +Checks added or extended: + +- `TestWebsiteSkillsSavedLifecycleAndColdRefusal`: locked replay with an empty + store and changed publisher bytes is refused by checksum, leaving state, lock + and the installed resource unchanged. A deleted historical resource fails cold + import with HTTP 404 and publishes no profile or lock. +- `TestWebSkillCancellationDuringFileBodyLeavesNoTree`: cancellation inside the + body read returns `context.Canceled` and removes the download workspace. +- `TestWebSkillRefusesUntrustedCertificateByDefault`: the default client refuses + a self-signed publisher certificate. +- `TestWebSkillSourceFactsBoundPinnedFileCount`: 256 pinned files are accepted + and 257 are refused. + +Commands and results (`GOTOOLCHAIN=go1.26.10`): + +- `cd cli && go test ./internal/agentpluginscli -run TestWebsiteSkillsSavedLifecycleAndColdRefusal -count=1`: PASS. +- `cd install/integrationctl/agentplugins && go test ./adapters/sourceacquisition -run TestWebSkill -count=5`: PASS. +- `cd cli && go test ./internal/agentpluginscli ./cmd/agentplugins/... -count=1`: PASS (`agentpluginscli` 332.5 s, `cmd/agentplugins` 67.0 s). +- `cd install/integrationctl/agentplugins && go test ./... -count=1`: FAIL in `adapters/clientdetect`, `providers`, `usecase` and `vscodelocalhooks`. The same 50 test failures reproduce on the unmodified base `19f12263` in a temporary worktree. `usecase` also panics in `TestPublicS1GeminiFailedFirstRetriesCompleteDesiredWithoutAdoptingForeignBytes` on this host. The cause of the base failures was not investigated. `cursorhooks` `TestLiteralArgvThroughJSONAndTargetShell` failed once in the full run and passed 3 of 3 reruns; it passed on the base. +- `golangci-lint` v2.13.2 with `--new-from-merge-base=origin/main` on the changed `cli` and `install/integrationctl/agentplugins` packages: 0 issues. The whole-file gate `--enable-only=revive,funlen,gocyclo,gocognit,dupl,depguard` on the same packages: 0 issues. + +The real-publisher receipt above predates this pass. Live acquisition was not +re-run after these changes; the changed code does not alter the success path of +acquisition. diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web.go index 0a3d15ce4..7bc6cd701 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web.go @@ -177,6 +177,9 @@ func (acquirer Acquirer) webSkillGET(ctx context.Context, endpoint string, limit } body, err := io.ReadAll(io.LimitReader(&archiveContextReader{ctx: ctx, reader: response.Body}, limit+1)) if err != nil { + if ctx.Err() != nil { + return nil, 0, "", ctx.Err() + } return nil, 0, "", errors.New("skill download was incomplete") } if int64(len(body)) > limit { diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web_test.go index ddf032026..aba09d772 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web_test.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web_test.go @@ -2,7 +2,9 @@ package sourceacquisition import ( "context" + "errors" "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -242,3 +244,74 @@ func TestWebSkillFallbackForPositiveHTMLOnly(t *testing.T) { }) } } + +// cancelAfterFirstRead cancels once the client has read the first body bytes, +// so cancellation lands inside the body read rather than inside Do. +type cancelAfterFirstRead struct { + io.ReadCloser + cancel context.CancelFunc +} + +func (body cancelAfterFirstRead) Read(buffer []byte) (int, error) { + n, err := body.ReadCloser.Read(buffer) + body.cancel() + return n, err +} + +type cancelOnFileBodyTransport struct { + base http.RoundTripper + path string + cancel context.CancelFunc +} + +func (transport cancelOnFileBodyTransport) RoundTrip(request *http.Request) (*http.Response, error) { + response, err := transport.base.RoundTrip(request) + if err == nil && request.URL.Path == transport.path { + response.Body = cancelAfterFirstRead{ReadCloser: response.Body, cancel: transport.cancel} + } + return response, err +} + +func TestWebSkillCancellationDuringFileBodyLeavesNoTree(t *testing.T) { + const prefix = "/publisher/.well-known/agent-skills/" + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case prefix + "index.json": + _, _ = fmt.Fprint(w, `{"skills":[{"name":"demo","files":["SKILL.md","references/slow.md"]}]}`) + case prefix + "demo/SKILL.md": + _, _ = fmt.Fprint(w, "inert") + case prefix + "demo/references/slow.md": + _, _ = fmt.Fprint(w, "partial") + w.(http.Flusher).Flush() + <-r.Context().Done() + default: + http.NotFound(w, r) + } + })) + defer server.Close() + temporary := t.TempDir() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := &http.Client{Transport: cancelOnFileBodyTransport{base: server.Client().Transport, path: prefix + "demo/references/slow.md", cancel: cancel}} + _, _, err := (Acquirer{TempRoot: temporary, HTTPClient: client}).ResolveHTTPSkill(ctx, server.URL+"/publisher", "demo") + if !errors.Is(err, context.Canceled) { + t.Fatalf("Skill file body cancellation = %v", err) + } + if entries, err := os.ReadDir(temporary); err != nil || len(entries) != 0 { + t.Fatalf("canceled Skill download left workspace: %v %v", entries, err) + } +} + +func TestWebSkillRefusesUntrustedCertificateByDefault(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `{"skills":[{"name":"demo","files":["SKILL.md"]}]}`) + })) + defer server.Close() + snapshot, _, err := (Acquirer{TempRoot: t.TempDir()}).ResolveHTTPSkill(context.Background(), server.URL, "demo") + if err == nil || !strings.Contains(err.Error(), "TLS") { + if err == nil { + _ = packagedigest.Remove(snapshot) + } + t.Fatalf("self-signed publisher certificate was not refused by TLS verification: %v", err) + } +} diff --git a/install/integrationctl/agentplugins/domain/web_skill_policy_test.go b/install/integrationctl/agentplugins/domain/web_skill_policy_test.go index b63735a1e..87af87f0a 100644 --- a/install/integrationctl/agentplugins/domain/web_skill_policy_test.go +++ b/install/integrationctl/agentplugins/domain/web_skill_policy_test.go @@ -1,6 +1,10 @@ package domain -import "testing" +import ( + "fmt" + "strings" + "testing" +) func TestWebSkillEndpointPreservesPrefixAndRejectsAmbiguousCoordinates(t *testing.T) { for input, expected := range map[string]string{"https://Skills.Example/": "https://skills.example", "https://Skills.Example/prefix/": "https://skills.example/prefix"} { @@ -35,3 +39,23 @@ func TestWebSkillBindingIdentityDependsOnPublisherAndName(t *testing.T) { t.Fatal("different publisher prefixes share a binding") } } + +func TestWebSkillSourceFactsBoundPinnedFileCount(t *testing.T) { + facts := func(count int) WebSkillSourceFacts { + hash := strings.Repeat("a", 64) + base := "https://skills.example" + pinned := WebSkillSourceFacts{URL: base, Skill: "pdf", IndexURL: base + "/.well-known/agent-skills/index.json", IndexSHA256: hash, Normalizer: "agent-skill-package/v1", RawTreeDigest: "sha256:" + hash} + pinned.Files = append(pinned.Files, WebSkillFile{Path: "SKILL.md", URL: base + "/.well-known/agent-skills/pdf/SKILL.md", SHA256: hash}) + for i := 1; i < count; i++ { + name := fmt.Sprintf("ref%03d.md", i) + pinned.Files = append(pinned.Files, WebSkillFile{Path: name, URL: base + "/.well-known/agent-skills/pdf/" + name, SHA256: hash}) + } + return pinned + } + if err := ValidateWebSkillSourceFacts(facts(256)); err != nil { + t.Fatalf("256 pinned files rejected: %v", err) + } + if err := ValidateWebSkillSourceFacts(facts(257)); err == nil { + t.Fatal("257 pinned files accepted") + } +} From cb3e89a55320cf9184a10c50f9c701dfc1a43fe1 Mon Sep 17 00:00:00 2001 From: iliya Date: Sat, 24 Oct 2026 20:46:01 +0300 Subject: [PATCH 24/25] fix(sources): expose website Skills in the production CLI - Forward HTTPS Skill acquisition through the production source acquirer, so website installs work in the real binary. Cover it with a composition test. - Refuse SCP-style Git hosts and users that start with a dash, and pass `--` before every remote Git endpoint. - Normalize the publisher URL once so the default alias and saved identity match. - Reuse one HTTP client per acquisition and bound the whole acquisition at 10 min while keeping a 30 s per-request limit. - Write Skill files through os.Root with O_EXCL. - Assert certificate verification errors by type; share one Skill-name rule. --- cli/cmd/agentplugins/main.go | 14 ++++ .../production_website_skill_test.go | 59 ++++++++++++++ .../agentpluginscli/skills_website.go | 8 +- .../website-skills-evidence.md | 33 +++++++- .../adapters/sourceacquisition/git_direct.go | 8 +- .../adapters/sourceacquisition/skill_web.go | 76 ++++++++++++++----- .../sourceacquisition/skill_web_test.go | 4 +- .../agentplugins/domain/web_skill_policy.go | 6 +- .../agentplugins/environment/skills.go | 12 +-- .../agentplugins/installer/acquired_skill.go | 16 +--- .../agentplugins/sourcespec/git.go | 4 +- .../sourcespec/git_option_injection_test.go | 13 ++++ 12 files changed, 196 insertions(+), 57 deletions(-) create mode 100644 cli/cmd/agentplugins/production_website_skill_test.go create mode 100644 install/integrationctl/agentplugins/sourcespec/git_option_injection_test.go diff --git a/cli/cmd/agentplugins/main.go b/cli/cmd/agentplugins/main.go index afbfcf652..031cd6051 100644 --- a/cli/cmd/agentplugins/main.go +++ b/cli/cmd/agentplugins/main.go @@ -327,3 +327,17 @@ func (acquirer lazySourceAcquirer) AcquireArchive(ctx context.Context, source so } return acquirer.acquirer.AcquireArchive(ctx, source, archiveSHA, tree) } + +func (acquirer lazySourceAcquirer) ResolveHTTPSkill(ctx context.Context, endpoint, exactName string) (domain.PackageSnapshot, domain.WebSkillSourceFacts, error) { + if err := acquirer.prepare(); err != nil { + return domain.PackageSnapshot{}, domain.WebSkillSourceFacts{}, err + } + return acquirer.acquirer.ResolveHTTPSkill(ctx, endpoint, exactName) +} + +func (acquirer lazySourceAcquirer) AcquireHTTPSkillVerified(ctx context.Context, facts domain.WebSkillSourceFacts) (domain.PackageSnapshot, error) { + if err := acquirer.prepare(); err != nil { + return domain.PackageSnapshot{}, err + } + return acquirer.acquirer.AcquireHTTPSkillVerified(ctx, facts) +} diff --git a/cli/cmd/agentplugins/production_website_skill_test.go b/cli/cmd/agentplugins/production_website_skill_test.go new file mode 100644 index 000000000..89a8c2fc6 --- /dev/null +++ b/cli/cmd/agentplugins/production_website_skill_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/adapters/packagedigest" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" +) + +// The production composition must expose HTTPS Skill acquisition. Trust is +// granted through the default transport, which the acquirer uses when no +// client is injected, so this exercises the real wrapper and no test double. +func TestProductionSourceAcquirerAcquiresHTTPSkills(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/.well-known/agent-skills/index.json": + _, _ = fmt.Fprint(w, `{"skills":[{"name":"demo","files":["SKILL.md"]}]}`) + case "/.well-known/agent-skills/demo/SKILL.md": + _, _ = fmt.Fprint(w, "---\nname: demo\ndescription: Test skill\n---\nInert content.\n") + default: + http.NotFound(w, r) + } + })) + defer server.Close() + previous := http.DefaultTransport + http.DefaultTransport = server.Client().Transport + t.Cleanup(func() { http.DefaultTransport = previous }) + + home := t.TempDir() + app, err := composeAgentpluginsApp(home, filepath.Join(home, "data")) + if err != nil { + t.Fatal(err) + } + acquirer, ok := app.SourceAcquirer.(interface { + ResolveHTTPSkill(context.Context, string, string) (domain.PackageSnapshot, domain.WebSkillSourceFacts, error) + AcquireHTTPSkillVerified(context.Context, domain.WebSkillSourceFacts) (domain.PackageSnapshot, error) + }) + if !ok { + t.Fatal("production source acquirer cannot acquire HTTPS Skills") + } + snapshot, facts, err := acquirer.ResolveHTTPSkill(context.Background(), server.URL, "demo") + if err != nil { + t.Fatal(err) + } + defer func() { _ = packagedigest.Remove(snapshot) }() + restored, err := acquirer.AcquireHTTPSkillVerified(context.Background(), facts) + if err != nil { + t.Fatal(err) + } + defer func() { _ = packagedigest.Remove(restored) }() + if restored.TreeDigest != snapshot.TreeDigest { + t.Fatal("locked restore changed the raw tree digest") + } +} diff --git a/cli/internal/agentpluginscli/skills_website.go b/cli/internal/agentpluginscli/skills_website.go index fb5b2f970..0ada49bdc 100644 --- a/cli/internal/agentpluginscli/skills_website.go +++ b/cli/internal/agentpluginscli/skills_website.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" "github.com/777genius/plugin-kit-ai/cli/internal/agentpluginscli/skillspicker" + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" ) // Website discovery is explicit and deterministic: a publisher index and exact @@ -26,9 +27,14 @@ func (selection *skillsAddOptions) runWebsite(cmd *cobra.Command, app App, opts if !interactive && !selection.yes && !opts.dryRun { return skillsMissingSelection(cmd, opts, "CONFIRMATION_REQUIRED", "Pass --yes to accept installation; --dry-run reviews without applying", []skillspicker.Candidate{}) } + // Normalize once so the saved identity, default alias and lock all use the same publisher URL. + website, err := domain.NormalizeWebSkillURL(endpoint) + if err != nil { + return err + } request := SkillsBatchRequest{} for _, name := range selection.names { - request.Selections = append(request.Selections, SkillsSelection{Website: endpoint, Name: name}) + request.Selections = append(request.Selections, SkillsSelection{Website: website, Name: name}) } for _, target := range targets { request.TargetIDs = append(request.TargetIDs, string(target)) diff --git a/docs/plans/source-transports-e2e/website-skills-evidence.md b/docs/plans/source-transports-e2e/website-skills-evidence.md index db45ec2a4..a2516e097 100644 --- a/docs/plans/source-transports-e2e/website-skills-evidence.md +++ b/docs/plans/source-transports-e2e/website-skills-evidence.md @@ -5,9 +5,13 @@ not qualify native agent execution or an authoring release. ## Lifecycle checks -`TestWebsiteSkillsSavedLifecycleAndColdRefusal` exercises the actual public CLI -composition against a disposable TLS publisher and isolated client roots. -Only the native client runner and security scanner use existing test fixtures. +`TestWebsiteSkillsSavedLifecycleAndColdRefusal` exercises the CLI command path +against a disposable TLS publisher and isolated client roots, with a raw +acquirer injected into the harness. It does not exercise the production +composition. `TestProductionSourceAcquirerAcquiresHTTPSkills` in +`cli/cmd/agentplugins` covers that: it resolves and restores a website Skill +through the acquirer that `composeAgentpluginsApp` returns. Only the native +client runner and security scanner use existing test fixtures. The extended scenario passed locally in 7.290 seconds: - Select and install one scoped website Skill, including actual nested resource @@ -116,3 +120,26 @@ Commands and results (`GOTOOLCHAIN=go1.26.10`): The real-publisher receipt above predates this pass. Live acquisition was not re-run after these changes; the changed code does not alter the success path of acquisition. + +## Independent review fix pass + +An independent pre-merge review of 4c3667ed returned REQUEST_CHANGES. Fixes: + +- Blocker: the production source acquirer did not expose HTTPS Skill methods, so + website installs failed in the real binary. The wrapper now forwards them, and + `TestProductionSourceAcquirerAcquiresHTTPSkills` covers the composed acquirer. +- Git option injection: SCP-style hosts and users starting with `-` are refused, + and Git receives `--` before every remote endpoint in the direct transport. + `TestNormalizeGitURLRejectsLeadingDashInSCPHostOrUser` covers it. +- Website URL is normalized once, so the default alias and saved identity match. +- One HTTP client per acquisition reuses connections; the per-request timeout + stays at 30 s and the whole acquisition is bounded at 10 min. +- Files are written through `os.Root` with `O_EXCL`. +- The certificate test asserts `tls.CertificateVerificationError`, not a message. +- The Skill-name rule has one implementation, `domain.ValidSkillName`. + +Commands and results (`GOTOOLCHAIN=go1.26.10`): + +- `install/integrationctl/agentplugins`: `go test ./domain ./environment ./installer ./sourcespec ./adapters/sourceacquisition ./adapters/dependencymanifest ./adapters/statev2 ./adapters/skillsource ./adapters/environmentstore -count=1`: PASS. +- `cli`: `go test ./cmd/agentplugins ./internal/agentpluginscli -count=1`: PASS (`agentpluginscli` 293.6 s). +- `golangci-lint` v2.13.2 with `--new-from-merge-base=origin/main` on the touched packages in both modules: 0 issues. The whole-file gate `--enable-only=revive,funlen,gocyclo,gocognit,dupl,depguard` on the same packages: 0 issues. diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go index ede731ab0..249307a16 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/git_direct.go @@ -152,7 +152,7 @@ func (acquirer Acquirer) ResolveGit(ctx context.Context, source sourcespec.Git, defer func() { _ = os.RemoveAll(workspace.root) }() endpoint = acquirer.directEndpoint(endpoint, repository) if !exactRef && (selector == "default" || selector == "HEAD") { - output, err := workspace.remote(ctx, "ls-remote", "--symref", endpoint, "HEAD") + output, err := workspace.remote(ctx, "ls-remote", "--symref", "--", endpoint, "HEAD") if err != nil { return RefResolution{}, directRefError(ctx) } @@ -175,7 +175,7 @@ func (acquirer Acquirer) ResolveGit(ctx context.Context, source sourcespec.Git, return RefResolution{}, errors.New("invalid exact Git ref") } } - output, err := workspace.remote(ctx, append([]string{"ls-remote", "--refs", endpoint}, patterns...)...) + output, err := workspace.remote(ctx, append([]string{"ls-remote", "--refs", "--", endpoint}, patterns...)...) if err != nil { return RefResolution{}, directRefError(ctx) } @@ -194,7 +194,7 @@ func (workspace directGitWorkspace) commit(ctx context.Context, endpoint string, if strings.HasPrefix(ref.name, "refs/heads/") { return ref.object, nil } - if _, err := workspace.remote(ctx, "fetch", "--quiet", "--depth=1", "--no-tags", "--no-recurse-submodules", endpoint, ref.object); err != nil { + if _, err := workspace.remote(ctx, "fetch", "--quiet", "--depth=1", "--no-tags", "--no-recurse-submodules", "--", endpoint, ref.object); err != nil { return "", directRefError(ctx) } output, err := workspace.run(ctx, false, "-C", workspace.repository, "rev-parse", "--verify", "FETCH_HEAD^{commit}") @@ -223,7 +223,7 @@ func (acquirer Acquirer) resolveBrowserGit(ctx context.Context, source sourcespe } defer func() { _ = os.RemoveAll(workspace.root) }() transportEndpoint := acquirer.directEndpoint(endpoint, repository) - output, err := workspace.remote(ctx, "ls-remote", "--refs", transportEndpoint, "refs/heads/*", "refs/tags/*") + output, err := workspace.remote(ctx, "ls-remote", "--refs", "--", transportEndpoint, "refs/heads/*", "refs/tags/*") if err != nil { return ResolvedGit{}, directRefError(ctx) } diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web.go index 7bc6cd701..9970a7f63 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web.go @@ -27,6 +27,10 @@ const ( webSkillTimeout = 30 * time.Second ) +// webSkillOperationTimeout bounds a whole acquisition: up to 256 files and 32 MiB +// can legitimately exceed a single request budget on a slow publisher. +const webSkillOperationTimeout = 10 * time.Minute + // ResolveHTTPSkill discovers exactly one named Skill and retains every listed // file. Its sealed snapshot contains the raw Skill, without generated metadata. // The caller owns the snapshot through packagedigest.Remove. @@ -36,13 +40,15 @@ func (acquirer Acquirer) ResolveHTTPSkill(ctx context.Context, endpoint, exactNa if err != nil { return domain.PackageSnapshot{}, facts, err } - ctx, cancel := context.WithTimeout(ctx, webSkillTimeout) + ctx, cancel := context.WithTimeout(ctx, webSkillOperationTimeout) defer cancel() + client := acquirer.webSkillClient() + defer client.CloseIdleConnections() indexURL := base + "/.well-known/agent-skills/index.json" - body, status, contentType, err := acquirer.webSkillGET(ctx, indexURL, webSkillIndexBytes) + body, status, contentType, err := webSkillGET(ctx, client, indexURL, webSkillIndexBytes) if err == nil && (status == http.StatusNotFound || status == http.StatusOK && webSkillHTML(body, contentType)) { indexURL = base + "/.well-known/skills/index.json" - body, status, _, err = acquirer.webSkillGET(ctx, indexURL, webSkillIndexBytes) + body, status, _, err = webSkillGET(ctx, client, indexURL, webSkillIndexBytes) } if err != nil { return domain.PackageSnapshot{}, facts, err @@ -69,7 +75,7 @@ func (acquirer Acquirer) ResolveHTTPSkill(ctx context.Context, endpoint, exactNa if err := validateWebSkillTreePaths(facts.Files); err != nil { return domain.PackageSnapshot{}, domain.WebSkillSourceFacts{}, err } - snapshot, err := acquirer.webSkillRetain(ctx, &facts, false) + snapshot, err := acquirer.webSkillRetain(ctx, client, &facts, false) if err != nil { return domain.PackageSnapshot{}, domain.WebSkillSourceFacts{}, err } @@ -88,20 +94,27 @@ func (acquirer Acquirer) AcquireHTTPSkillVerified(ctx context.Context, facts dom } // Avoid changing a caller-owned files slice while checking retained pins. facts.Files = append([]domain.WebSkillFile(nil), facts.Files...) - ctx, cancel := context.WithTimeout(ctx, webSkillTimeout) + ctx, cancel := context.WithTimeout(ctx, webSkillOperationTimeout) defer cancel() - return acquirer.webSkillRetain(ctx, &facts, true) + client := acquirer.webSkillClient() + defer client.CloseIdleConnections() + return acquirer.webSkillRetain(ctx, client, &facts, true) } -func (acquirer Acquirer) webSkillRetain(ctx context.Context, facts *domain.WebSkillSourceFacts, verify bool) (domain.PackageSnapshot, error) { +func (acquirer Acquirer) webSkillRetain(ctx context.Context, client *http.Client, facts *domain.WebSkillSourceFacts, verify bool) (domain.PackageSnapshot, error) { workspace, err := os.MkdirTemp(acquirer.TempRoot, "agentplugins-web-skill-*") if err != nil { return domain.PackageSnapshot{}, errors.New("create private Skill workspace") } defer func() { _ = os.RemoveAll(workspace) }() + root, err := os.OpenRoot(workspace) + if err != nil { + return domain.PackageSnapshot{}, errors.New("open private Skill workspace") + } + defer func() { _ = root.Close() }() var total int64 for i, file := range facts.Files { - body, status, _, err := acquirer.webSkillGET(ctx, file.URL, min(webSkillFileBytes, webSkillTreeBytes-total)) + body, status, _, err := webSkillGET(ctx, client, file.URL, min(webSkillFileBytes, webSkillTreeBytes-total)) if err != nil { return domain.PackageSnapshot{}, err } @@ -114,12 +127,8 @@ func (acquirer Acquirer) webSkillRetain(ctx context.Context, facts *domain.WebSk return domain.PackageSnapshot{}, errors.New("skill file original-byte SHA256 does not match its locked checksum") } facts.Files[i].SHA256 = checksum - filename := filepath.Join(workspace, filepath.FromSlash(file.Path)) - if err := os.MkdirAll(filepath.Dir(filename), 0o700); err != nil { - return domain.PackageSnapshot{}, errors.New("create private Skill file parent") - } - if err := os.WriteFile(filename, body, 0o600); err != nil { - return domain.PackageSnapshot{}, errors.New("write private Skill file") + if err := writeWebSkillFile(root, filepath.FromSlash(file.Path), body); err != nil { + return domain.PackageSnapshot{}, err } } snapshot, err := acquirer.digester().SnapshotWithExecutables(ctx, workspace, domain.SourceIdentity{}, nil) @@ -146,24 +155,51 @@ func webSkillSHA256(body []byte) string { return hex.EncodeToString(digest[:]) } +// webSkillClient is built once per acquisition, so every file request reuses +// the same connection pool. Redirects are refused and each request is bounded. +func (acquirer Acquirer) webSkillClient() *http.Client { + client := acquirer.archiveHTTPClient() + client.Timeout = min(client.Timeout, webSkillTimeout) + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + return client +} + +// writeWebSkillFile creates a new file inside the workspace root. O_EXCL stops a +// colliding name from silently replacing an earlier file on any filesystem. +func writeWebSkillFile(root *os.Root, relative string, body []byte) error { + if dir := filepath.Dir(relative); dir != "." { + if err := root.MkdirAll(dir, 0o700); err != nil { + return errors.New("create private Skill file parent") + } + } + file, err := root.OpenFile(relative, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return errors.New("write private Skill file") + } + if _, err := file.Write(body); err != nil { + _ = file.Close() + return errors.New("write private Skill file") + } + if err := file.Close(); err != nil { + return errors.New("write private Skill file") + } + return nil +} + // webSkillGET inherits only the host's TLS/proxy transport. Cookies, automatic // authentication, content encodings and all redirects are deliberately refused. -func (acquirer Acquirer) webSkillGET(ctx context.Context, endpoint string, limit int64) ([]byte, int, string, error) { +func webSkillGET(ctx context.Context, client *http.Client, endpoint string, limit int64) ([]byte, int, string, error) { request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody) if err != nil { return nil, 0, "", errors.New("create Skill HTTPS request") } request.Header.Set("Accept-Encoding", "identity") - client := acquirer.archiveHTTPClient() - defer client.CloseIdleConnections() - client.Timeout = min(client.Timeout, webSkillTimeout) - client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } response, err := client.Do(request) if err != nil { if ctx.Err() != nil { return nil, 0, "", ctx.Err() } - return nil, 0, "", errors.New("skill HTTPS request failed (check TLS and connectivity)") + return nil, 0, "", fmt.Errorf("skill HTTPS request failed (check TLS and connectivity): %w", err) } defer func() { _ = response.Body.Close() }() if response.StatusCode != http.StatusOK { diff --git a/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web_test.go b/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web_test.go index aba09d772..22080ad7d 100644 --- a/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web_test.go +++ b/install/integrationctl/agentplugins/adapters/sourceacquisition/skill_web_test.go @@ -2,6 +2,7 @@ package sourceacquisition import ( "context" + "crypto/tls" "errors" "fmt" "io" @@ -308,7 +309,8 @@ func TestWebSkillRefusesUntrustedCertificateByDefault(t *testing.T) { })) defer server.Close() snapshot, _, err := (Acquirer{TempRoot: t.TempDir()}).ResolveHTTPSkill(context.Background(), server.URL, "demo") - if err == nil || !strings.Contains(err.Error(), "TLS") { + var verification *tls.CertificateVerificationError + if err == nil || !errors.As(err, &verification) { if err == nil { _ = packagedigest.Remove(snapshot) } diff --git a/install/integrationctl/agentplugins/domain/web_skill_policy.go b/install/integrationctl/agentplugins/domain/web_skill_policy.go index db3dbb6c6..09cc0ad1d 100644 --- a/install/integrationctl/agentplugins/domain/web_skill_policy.go +++ b/install/integrationctl/agentplugins/domain/web_skill_policy.go @@ -57,7 +57,9 @@ func validateWebSkillEndpointPath(u *url.URL) error { return nil } -func validWebSkillName(name string) bool { +// ValidSkillName is the one Skill-name contract shared by every acquisition, +// environment and installer path: 1-64 lowercase letters, digits and single hyphens. +func ValidSkillName(name string) bool { if size := utf8.RuneCountInString(name); size < 1 || size > 64 || !utf8.ValidString(name) || strings.HasPrefix(name, "-") || strings.HasSuffix(name, "-") || strings.Contains(name, "--") { return false } @@ -104,7 +106,7 @@ func ValidateWebSkillSourceFacts(facts WebSkillSourceFacts) error { } func validWebSkillPinIdentity(facts WebSkillSourceFacts) bool { - return validWebSkillName(facts.Skill) && facts.Normalizer == "agent-skill-package/v1" && webSkillSHA256.MatchString(facts.IndexSHA256) && strings.HasPrefix(facts.RawTreeDigest, "sha256:") && webSkillSHA256.MatchString(strings.TrimPrefix(facts.RawTreeDigest, "sha256:")) + return ValidSkillName(facts.Skill) && facts.Normalizer == "agent-skill-package/v1" && webSkillSHA256.MatchString(facts.IndexSHA256) && strings.HasPrefix(facts.RawTreeDigest, "sha256:") && webSkillSHA256.MatchString(strings.TrimPrefix(facts.RawTreeDigest, "sha256:")) } func validateWebSkillPinnedFiles(facts WebSkillSourceFacts) error { diff --git a/install/integrationctl/agentplugins/environment/skills.go b/install/integrationctl/agentplugins/environment/skills.go index aae8d6226..2ee74385d 100644 --- a/install/integrationctl/agentplugins/environment/skills.go +++ b/install/integrationctl/agentplugins/environment/skills.go @@ -2,9 +2,8 @@ package environment import ( "errors" - "strings" - "unicode" - "unicode/utf8" + + "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" ) func validateSkillSource(source *GitSource) error { @@ -24,14 +23,9 @@ func validateSkillSource(source *GitSource) error { } func validateSkillName(name string) error { - if n := utf8.RuneCountInString(name); n < 1 || n > 64 || strings.HasPrefix(name, "-") || strings.HasSuffix(name, "-") || strings.Contains(name, "--") { + if !domain.ValidSkillName(name) { return errors.New("invalid expected Skill name") } - for _, r := range name { - if r != '-' && !unicode.IsLower(r) && !unicode.IsNumber(r) { - return errors.New("invalid expected Skill name") - } - } return nil } diff --git a/install/integrationctl/agentplugins/installer/acquired_skill.go b/install/integrationctl/agentplugins/installer/acquired_skill.go index a164e2d21..4354e82bb 100644 --- a/install/integrationctl/agentplugins/installer/acquired_skill.go +++ b/install/integrationctl/agentplugins/installer/acquired_skill.go @@ -3,8 +3,6 @@ package installer import ( "path" "strings" - "unicode" - "unicode/utf8" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/sourcespec" @@ -18,7 +16,7 @@ func acquiredSkillSourceKey(source domain.SourceIdentity, portable bool) (string return "", ErrInvalidRequest } _, repository, err := sourcespec.NormalizeGitURL("https://github.com/" + source.Repository + ".git") - if err != nil || !strings.EqualFold(repository, source.Repository) || !validAcquiredSkillName(facts.Name) { + if err != nil || !strings.EqualFold(repository, source.Repository) || !domain.ValidSkillName(facts.Name) { return "", ErrInvalidRequest } if facts.Normalizer != "agent-skill-package/v1" || !strings.HasPrefix(facts.RawTreeDigest, "sha256:") || !sourcespec.ValidSHA256(strings.TrimPrefix(facts.RawTreeDigest, "sha256:")) { @@ -54,15 +52,3 @@ func acquiredWebSkillSourceKey(source domain.SourceIdentity, portable bool) (str } return key, nil } - -func validAcquiredSkillName(name string) bool { - if size := utf8.RuneCountInString(name); size < 1 || size > 64 || strings.HasPrefix(name, "-") || strings.HasSuffix(name, "-") || strings.Contains(name, "--") { - return false - } - for _, character := range name { - if character != '-' && !unicode.IsLower(character) && !unicode.IsNumber(character) { - return false - } - } - return true -} diff --git a/install/integrationctl/agentplugins/sourcespec/git.go b/install/integrationctl/agentplugins/sourcespec/git.go index 8e36780b0..81cdb4f18 100644 --- a/install/integrationctl/agentplugins/sourcespec/git.go +++ b/install/integrationctl/agentplugins/sourcespec/git.go @@ -62,7 +62,7 @@ func normalizeGitUser(u *url.URL) error { func normalizeSCP(value string) (string, string, error) { left, right, ok := strings.Cut(value, ":") - if !ok || right == "" || strings.HasPrefix(right, "-") || strings.ContainsAny(left, " /?#") || strings.ContainsAny(right, "?#") || (!strings.Contains(left, "@") && !strings.Contains(left, ".")) { + if !ok || right == "" || strings.HasPrefix(right, "-") || strings.HasPrefix(left, "-") || strings.ContainsAny(left, " /?#") || strings.ContainsAny(right, "?#") || (!strings.Contains(left, "@") && !strings.Contains(left, ".")) { return "", "", errors.New("Git endpoint must use HTTPS, ssh:// or user@host:path") } user, host, hasUser := strings.Cut(left, "@") @@ -74,7 +74,7 @@ func normalizeSCP(value string) (string, string, error) { host = user user = "" } - if strings.ContainsAny(host, "[]: ") || strings.ContainsAny(user, ": ") { + if strings.ContainsAny(host, "[]: ") || strings.ContainsAny(user, ": ") || strings.HasPrefix(host, "-") { return "", "", errors.New("invalid SSH endpoint") } if err := validateSCPPath(right); err != nil { diff --git a/install/integrationctl/agentplugins/sourcespec/git_option_injection_test.go b/install/integrationctl/agentplugins/sourcespec/git_option_injection_test.go new file mode 100644 index 000000000..26ff36698 --- /dev/null +++ b/install/integrationctl/agentplugins/sourcespec/git_option_injection_test.go @@ -0,0 +1,13 @@ +package sourcespec + +import "testing" + +// A leading dash in an SCP-style host or user would reach git as an option +// (for example --upload-pack) when the endpoint is passed positionally. +func TestNormalizeGitURLRejectsLeadingDashInSCPHostOrUser(t *testing.T) { + for _, endpoint := range []string{"--upload-pack=touch.pwned:x", "-x@host.example:repo.git", "user@-host.example:repo.git"} { + if _, _, err := NormalizeGitURL(endpoint); err == nil { + t.Fatalf("option-like SCP endpoint accepted: %q", endpoint) + } + } +} From 00645636b0297ab23d2d24a3fd1e2e5bf8beb330 Mon Sep 17 00:00:00 2001 From: iliya Date: Sat, 24 Oct 2026 21:05:35 +0300 Subject: [PATCH 25/25] fix(sources): close review notes on website Skill identity and Git endpoints - Refuse ssh:// hosts and users that start with a dash, as the SCP form does. - Use domain.ValidSkillName in conformance instead of a fourth copy. - Derive the default alias from the normalized publisher URL. - Document the Unicode behaviour of the shared Skill-name rule. --- .../agentpluginscli/skills_batch_selection.go | 4 +++- .../agentpluginscli/skills_website_alias_test.go | 13 +++++++++++++ .../website-skills-evidence.md | 13 +++++++++++++ .../agentplugins/conformance/skills.go | 12 +----------- .../agentplugins/domain/web_skill_policy.go | 3 ++- .../integrationctl/agentplugins/sourcespec/git.go | 3 +++ .../sourcespec/git_option_injection_test.go | 2 +- 7 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 cli/internal/agentpluginscli/skills_website_alias_test.go diff --git a/cli/internal/agentpluginscli/skills_batch_selection.go b/cli/internal/agentpluginscli/skills_batch_selection.go index 82d10cf1a..c0ce2e116 100644 --- a/cli/internal/agentpluginscli/skills_batch_selection.go +++ b/cli/internal/agentpluginscli/skills_batch_selection.go @@ -122,7 +122,9 @@ func defaultSkillAlias(selection SkillsSelection) string { probe := environment.Requirement{Alias: selection.Name, Git: &environment.GitSource{Repository: selection.Repository, Selector: environment.Selector{Kind: environment.SelectorDefault}}} if selection.Website != "" { probe.Git = nil - probe.Web = &environment.WebSource{URL: selection.Website, Skill: selection.Name} + // An invalid publisher URL normalizes to "", which fails Validate and keeps the hashed alias. + endpoint, _ := environment.NormalizeWebURL(selection.Website) + probe.Web = &environment.WebSource{URL: endpoint, Skill: selection.Name} } if probe.Validate() == nil { return selection.Name diff --git a/cli/internal/agentpluginscli/skills_website_alias_test.go b/cli/internal/agentpluginscli/skills_website_alias_test.go new file mode 100644 index 000000000..3b251f0a9 --- /dev/null +++ b/cli/internal/agentpluginscli/skills_website_alias_test.go @@ -0,0 +1,13 @@ +package agentpluginscli + +import "testing" + +// A publisher URL that is not in canonical form must still derive the Skill +// name as the default alias, the same as the canonical URL does. +func TestWebsiteDefaultAliasUsesCanonicalPublisherURL(t *testing.T) { + for _, website := range []string{"https://publisher.example/docs", "https://Publisher.Example/docs/"} { + if got := defaultSkillAlias(SkillsSelection{Website: website, Name: "pdf"}); got != "pdf" { + t.Fatalf("alias for %q = %q, want the Skill name", website, got) + } + } +} diff --git a/docs/plans/source-transports-e2e/website-skills-evidence.md b/docs/plans/source-transports-e2e/website-skills-evidence.md index a2516e097..6399657a8 100644 --- a/docs/plans/source-transports-e2e/website-skills-evidence.md +++ b/docs/plans/source-transports-e2e/website-skills-evidence.md @@ -143,3 +143,16 @@ Commands and results (`GOTOOLCHAIN=go1.26.10`): - `install/integrationctl/agentplugins`: `go test ./domain ./environment ./installer ./sourcespec ./adapters/sourceacquisition ./adapters/dependencymanifest ./adapters/statev2 ./adapters/skillsource ./adapters/environmentstore -count=1`: PASS. - `cli`: `go test ./cmd/agentplugins ./internal/agentpluginscli -count=1`: PASS (`agentpluginscli` 293.6 s). - `golangci-lint` v2.13.2 with `--new-from-merge-base=origin/main` on the touched packages in both modules: 0 issues. The whole-file gate `--enable-only=revive,funlen,gocyclo,gocognit,dupl,depguard` on the same packages: 0 issues. + +A focused re-review of `cb3e89a5` returned APPROVE. Its four P3 notes are fixed in the next commit: + +- `ssh://` hosts and users starting with a dash are refused, not only the SCP form. +- `conformance` delegates the Skill-name check to `domain.ValidSkillName`. +- The `ValidSkillName` comment states the Unicode behaviour. +- The default alias normalizes the publisher URL, covered by `TestWebsiteDefaultAliasUsesCanonicalPublisherURL`. + +Final commands (`GOTOOLCHAIN=go1.26.10`): + +- `install/integrationctl/agentplugins`: `go test ./adapters/sourceacquisition ./adapters/skillsource ./adapters/dependencymanifest ./adapters/statev2 ./adapters/environmentstore ./conformance ./domain ./environment ./installer ./sourcespec -count=1`: PASS. +- `cli`: `go test ./cmd/agentplugins ./internal/agentpluginscli -count=1`: PASS (`agentpluginscli` 288.3 s). +- Lint and whole-file gate on the touched packages in both modules: 0 issues. diff --git a/install/integrationctl/agentplugins/conformance/skills.go b/install/integrationctl/agentplugins/conformance/skills.go index fd800a9c3..8fa97a0de 100644 --- a/install/integrationctl/agentplugins/conformance/skills.go +++ b/install/integrationctl/agentplugins/conformance/skills.go @@ -7,7 +7,6 @@ import ( "fmt" "regexp" "strings" - "unicode" "unicode/utf8" "github.com/777genius/plugin-kit-ai/install/integrationctl/agentplugins/domain" @@ -175,16 +174,7 @@ func skillError(code, format string, args ...any) error { return &skillFailure{code, fmt.Sprintf(format, args...)} } func normativeSkillName(name string) bool { - n := utf8.RuneCountInString(name) - if n < 1 || n > MaxSkillNameBytes || strings.HasPrefix(name, "-") || strings.HasSuffix(name, "-") || strings.Contains(name, "--") { - return false - } - for _, r := range name { - if r != '-' && !unicode.IsLower(r) && !unicode.IsNumber(r) { - return false - } - } - return true + return domain.ValidSkillName(name) } // SkillErrorCode distinguishes normative invalidity from incomplete host-safety diff --git a/install/integrationctl/agentplugins/domain/web_skill_policy.go b/install/integrationctl/agentplugins/domain/web_skill_policy.go index 09cc0ad1d..90b11dc7d 100644 --- a/install/integrationctl/agentplugins/domain/web_skill_policy.go +++ b/install/integrationctl/agentplugins/domain/web_skill_policy.go @@ -58,7 +58,8 @@ func validateWebSkillEndpointPath(u *url.URL) error { } // ValidSkillName is the one Skill-name contract shared by every acquisition, -// environment and installer path: 1-64 lowercase letters, digits and single hyphens. +// environment and installer path: 1-64 runes that are Unicode lowercase letters +// or numbers, with single hyphens between them and no leading or trailing hyphen. func ValidSkillName(name string) bool { if size := utf8.RuneCountInString(name); size < 1 || size > 64 || !utf8.ValidString(name) || strings.HasPrefix(name, "-") || strings.HasSuffix(name, "-") || strings.Contains(name, "--") { return false diff --git a/install/integrationctl/agentplugins/sourcespec/git.go b/install/integrationctl/agentplugins/sourcespec/git.go index 81cdb4f18..21dc8b3ed 100644 --- a/install/integrationctl/agentplugins/sourcespec/git.go +++ b/install/integrationctl/agentplugins/sourcespec/git.go @@ -29,6 +29,9 @@ func NormalizeGitURL(value string) (string, string, error) { if err := normalizeGitUser(u); err != nil { return "", "", err } + if strings.HasPrefix(u.Hostname(), "-") || (u.User != nil && strings.HasPrefix(u.User.Username(), "-")) { + return "", "", errors.New("Git endpoint host and user cannot start with a dash") + } repository := "" if u.Scheme == "https" && u.Host == "github.com" { if encodedSeparator(u.RawPath) { diff --git a/install/integrationctl/agentplugins/sourcespec/git_option_injection_test.go b/install/integrationctl/agentplugins/sourcespec/git_option_injection_test.go index 26ff36698..8c296660f 100644 --- a/install/integrationctl/agentplugins/sourcespec/git_option_injection_test.go +++ b/install/integrationctl/agentplugins/sourcespec/git_option_injection_test.go @@ -5,7 +5,7 @@ import "testing" // A leading dash in an SCP-style host or user would reach git as an option // (for example --upload-pack) when the endpoint is passed positionally. func TestNormalizeGitURLRejectsLeadingDashInSCPHostOrUser(t *testing.T) { - for _, endpoint := range []string{"--upload-pack=touch.pwned:x", "-x@host.example:repo.git", "user@-host.example:repo.git"} { + for _, endpoint := range []string{"--upload-pack=touch.pwned:x", "-x@host.example:repo.git", "user@-host.example:repo.git", "ssh://-oProxyCommand=x/repo.git", "ssh://-x@host.example/repo.git"} { if _, _, err := NormalizeGitURL(endpoint); err == nil { t.Fatalf("option-like SCP endpoint accepted: %q", endpoint) }