From 64c6f8521739f67f730c067ca96250531e6e3329 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 11 Nov 2026 09:27:53 -0700 Subject: [PATCH 1/9] =?UTF-8?q?=E2=9C=A8=20workflowmodel:=20signed=20bundl?= =?UTF-8?q?e=20format=20and=20embedded=20CNCF=20corpus?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add pkg/workflowmodel/bundle: Export compiles a model and wraps it with its profile and a sha256 of the canonical model JSON; Encode/Decode use canonical (sorted-key) JSON; Verify recomputes the hash, recompiles the model locally and compares the profile structurally so importers never trust a published profile. Add pkg/workflowmodel/ingest/corpus embedding the CNCF survey corpus (262 projects / 299 repos) so the hub marketplace refresh and cmd/workflow-survey share one list; the survey falls back to it when a corpus directory has no projects.json. Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --- src/cmd/workflow-survey/main.go | 5 +- src/cmd/workflow-survey/main_test.go | 4 +- src/pkg/workflowmodel/bundle/bundle.go | 294 +++ src/pkg/workflowmodel/bundle/bundle_test.go | 234 ++ src/pkg/workflowmodel/ingest/corpus/corpus.go | 90 + .../workflowmodel/ingest/corpus/corpus.json | 2133 +++++++++++++++++ .../ingest/corpus/corpus_test.go | 60 + 7 files changed, 2817 insertions(+), 3 deletions(-) create mode 100644 src/pkg/workflowmodel/bundle/bundle.go create mode 100644 src/pkg/workflowmodel/bundle/bundle_test.go create mode 100644 src/pkg/workflowmodel/ingest/corpus/corpus.go create mode 100644 src/pkg/workflowmodel/ingest/corpus/corpus.json create mode 100644 src/pkg/workflowmodel/ingest/corpus/corpus_test.go diff --git a/src/cmd/workflow-survey/main.go b/src/cmd/workflow-survey/main.go index 50603b28e6..692c3f2901 100644 --- a/src/cmd/workflow-survey/main.go +++ b/src/cmd/workflow-survey/main.go @@ -19,6 +19,7 @@ import ( "github.com/hivecommons/hive/pkg/workflowmodel" "github.com/hivecommons/hive/pkg/workflowmodel/ingest" + corpuslist "github.com/hivecommons/hive/pkg/workflowmodel/ingest/corpus" "github.com/hivecommons/hive/pkg/workflowmodel/render" ) @@ -168,11 +169,13 @@ func realMain(args []string, stdout, stderr io.Writer) int { // loadProjects maps owner/repo to project maturity from the collector's // projects.json, when present. +// loadProjects reads /projects.json; when the corpus directory +// carries none it falls back to the embedded CNCF list (#17620 phase 7). func loadProjects(corpus string) map[string]string { out := map[string]string{} b, err := os.ReadFile(filepath.Join(corpus, "projects.json")) if err != nil { - return out + return corpuslist.Maturity() } var projects []struct { Project string `json:"project"` diff --git a/src/cmd/workflow-survey/main_test.go b/src/cmd/workflow-survey/main_test.go index 6703b3fbc6..6a939a1a45 100644 --- a/src/cmd/workflow-survey/main_test.go +++ b/src/cmd/workflow-survey/main_test.go @@ -253,8 +253,8 @@ func TestHelpers(t *testing.T) { if got := sortedKeys(map[string]int{"b": 1, "a": 2}); strings.Join(got, ",") != "a,b" { t.Errorf("sortedKeys = %v", got) } - if loadProjects(t.TempDir()) == nil { - t.Error("loadProjects without projects.json should return an empty map") + if m := loadProjects(t.TempDir()); m["project-akri/akri"] != "sandbox" { + t.Errorf("loadProjects without projects.json should fall back to the embedded corpus, got %d entries", len(m)) } dir := t.TempDir() if err := os.WriteFile(filepath.Join(dir, "projects.json"), []byte("{not json"), 0o644); err != nil { diff --git a/src/pkg/workflowmodel/bundle/bundle.go b/src/pkg/workflowmodel/bundle/bundle.go new file mode 100644 index 0000000000..90e0701260 --- /dev/null +++ b/src/pkg/workflowmodel/bundle/bundle.go @@ -0,0 +1,294 @@ +// Package bundle defines the portable interchange format for workflow models +// (#17620 phase 7). A bundle carries a model, the profile the publisher +// compiled from it, and a sha256 of the canonical model JSON so a receiving +// spoke can verify the model was not altered in transit and that the published +// profile is exactly what a local compile produces. Importers never trust the +// published profile: Verify recompiles and compares structurally, and callers +// are expected to recompile again when they store the model. +// +// The package depends only on pkg/workflowmodel so it can be shared by the +// spoke dashboard, the hub marketplace and CLI tooling. +package bundle + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/workflowmodel" +) + +// APIVersion identifies the bundle envelope format. +const APIVersion = "workflow.hivecommons.dev/v1alpha1" + +// Kind is the envelope kind. +const Kind = "WorkflowBundle" + +// Provenance tiers a bundle may carry. They extend the model provenance +// vocabulary with the marketplace-only `local-published` tier used for +// bundles a spoke publishes from its own library. +const ( + TierBuiltIn = string(workflowmodel.ProvenanceBuiltIn) + TierCurated = string(workflowmodel.ProvenanceCurated) + TierIngestedUnreviewed = string(workflowmodel.ProvenanceIngestedUnreviewed) + TierLocal = string(workflowmodel.ProvenanceLocal) + TierLocalPublished = "local-published" +) + +// Errors returned by Verify and Decode. +var ( + ErrBadVersion = errors.New("bundle: unsupported apiVersion") + ErrBadKind = errors.New("bundle: unsupported kind") + ErrMissingModel = errors.New("bundle: missing model") + ErrMissingProfile = errors.New("bundle: missing profile") + ErrHashMismatch = errors.New("bundle: hash does not match canonical model") + ErrProfileMismatch = errors.New("bundle: published profile does not match local compile") + ErrBadTier = errors.New("bundle: unknown provenance tier") +) + +// Bundle is the portable envelope. +type Bundle struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Model *workflowmodel.Model `json:"model"` + Profile *workflowmodel.Profile `json:"profile"` + // Hash is the sha256 (hex) of the canonical model JSON; see CanonicalHash. + Hash string `json:"hash"` + CreatedAt time.Time `json:"createdAt"` + CreatedBy string `json:"createdBy,omitempty"` + // Tier is the provenance tier the publisher asserts. Receivers apply + // their own policy (the hub only honours curated/built-in from admins). + Tier string `json:"tier,omitempty"` +} + +// Key is the catalogue key `name@version`. +func (b *Bundle) Key() string { + if b == nil || b.Model == nil { + return "" + } + return b.Model.Metadata.Name + "@" + b.Model.Spec.Meta.Version +} + +// KnownTier reports whether t is a provenance tier the bundle format defines. +func KnownTier(t string) bool { + switch t { + case TierBuiltIn, TierCurated, TierIngestedUnreviewed, TierLocal, TierLocalPublished: + return true + } + return false +} + +// Export validates and compiles m, then wraps it with its profile and hash. +// It refuses models that do not compile, so a bundle always carries a +// profile consistent with its model at creation time. +func Export(m *workflowmodel.Model, by string, now time.Time) (*Bundle, error) { + if m == nil { + return nil, ErrMissingModel + } + if err := m.Validate(); err != nil { + return nil, err + } + profile, _, err := workflowmodel.Compile(m) + if err != nil { + return nil, err + } + tier := strings.TrimSpace(string(m.Spec.Meta.Provenance)) + if tier == "" { + tier = TierLocal + } + if !KnownTier(tier) { + return nil, fmt.Errorf("%w: %q", ErrBadTier, tier) + } + return &Bundle{ + APIVersion: APIVersion, + Kind: Kind, + Model: m, + Profile: profile, + Hash: CanonicalHash(m), + CreatedAt: now.UTC(), + CreatedBy: strings.TrimSpace(by), + Tier: tier, + }, nil +} + +// Encode serialises b as canonical JSON (object keys sorted recursively, no +// insignificant whitespace), so equal bundles produce byte-identical output +// and hashes over encoded bundles are stable. +func Encode(b *Bundle) ([]byte, error) { + if b == nil { + return nil, ErrMissingModel + } + return canonicalJSON(b) +} + +// Decode parses a bundle and checks the envelope fields. It does not verify +// hashes or profiles; call Verify for that. +func Decode(data []byte) (*Bundle, error) { + var b Bundle + dec := json.NewDecoder(bytes.NewReader(data)) + dec.DisallowUnknownFields() + if err := dec.Decode(&b); err != nil { + return nil, fmt.Errorf("bundle: decode: %w", err) + } + if b.APIVersion != APIVersion { + return nil, fmt.Errorf("%w: %q", ErrBadVersion, b.APIVersion) + } + if b.Kind != "" && b.Kind != Kind { + return nil, fmt.Errorf("%w: %q", ErrBadKind, b.Kind) + } + b.Kind = Kind + if b.Model == nil { + return nil, ErrMissingModel + } + if b.Profile == nil { + return nil, ErrMissingProfile + } + return &b, nil +} + +// Verify checks that the bundle is internally consistent: the envelope is +// well-formed, the hash matches the canonical model JSON, the model validates +// and compiles locally, and the compiled profile structurally equals the +// published one. A mismatch is reported as ErrProfileMismatch. +func Verify(b *Bundle) error { + if b == nil || b.Model == nil { + return ErrMissingModel + } + if b.APIVersion != APIVersion { + return fmt.Errorf("%w: %q", ErrBadVersion, b.APIVersion) + } + if b.Kind != "" && b.Kind != Kind { + return fmt.Errorf("%w: %q", ErrBadKind, b.Kind) + } + if b.Profile == nil { + return ErrMissingProfile + } + if b.Tier != "" && !KnownTier(b.Tier) { + return fmt.Errorf("%w: %q", ErrBadTier, b.Tier) + } + if got := CanonicalHash(b.Model); got != b.Hash { + return fmt.Errorf("%w (have %s, want %s)", ErrHashMismatch, short(b.Hash), short(got)) + } + if err := b.Model.Validate(); err != nil { + return err + } + local, _, err := workflowmodel.Compile(b.Model) + if err != nil { + return err + } + want, err := canonicalJSON(local) + if err != nil { + return err + } + have, err := canonicalJSON(b.Profile) + if err != nil { + return err + } + if !bytes.Equal(want, have) { + return ErrProfileMismatch + } + return nil +} + +// CanonicalHash is the sha256 (hex) of the canonical JSON of the model's +// spec and metadata, independent of key order or formatting in the source +// document. It differs from Model.Hash, which covers the spec only, because +// the catalogue key includes the name. +func CanonicalHash(m *workflowmodel.Model) string { + if m == nil { + return "" + } + data, err := canonicalJSON(struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata workflowmodel.Metadata `json:"metadata"` + Spec workflowmodel.Spec `json:"spec"` + }{m.APIVersion, m.Kind, m.Metadata, m.Spec}) + if err != nil { + return "" + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +// canonicalJSON marshals v, re-decodes into generic values and re-encodes +// with sorted keys and no whitespace. Go's encoding/json already sorts map +// keys, so a round trip through interface{} normalises struct field order. +func canonicalJSON(v interface{}) ([]byte, error) { + raw, err := json.Marshal(v) + if err != nil { + return nil, err + } + var generic interface{} + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&generic); err != nil { + return nil, err + } + var buf bytes.Buffer + if err := writeCanonical(&buf, generic); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func writeCanonical(buf *bytes.Buffer, v interface{}) error { + switch t := v.(type) { + case map[string]interface{}: + keys := make([]string, 0, len(t)) + for k := range t { + keys = append(keys, k) + } + sort.Strings(keys) + buf.WriteByte('{') + for i, k := range keys { + if i > 0 { + buf.WriteByte(',') + } + kb, err := json.Marshal(k) + if err != nil { + return err + } + buf.Write(kb) + buf.WriteByte(':') + if err := writeCanonical(buf, t[k]); err != nil { + return err + } + } + buf.WriteByte('}') + case []interface{}: + buf.WriteByte('[') + for i, e := range t { + if i > 0 { + buf.WriteByte(',') + } + if err := writeCanonical(buf, e); err != nil { + return err + } + } + buf.WriteByte(']') + default: + b, err := json.Marshal(t) + if err != nil { + return err + } + buf.Write(b) + } + return nil +} + +func short(h string) string { + if len(h) > 12 { + return h[:12] + } + if h == "" { + return "" + } + return h +} diff --git a/src/pkg/workflowmodel/bundle/bundle_test.go b/src/pkg/workflowmodel/bundle/bundle_test.go new file mode 100644 index 0000000000..f25f98b787 --- /dev/null +++ b/src/pkg/workflowmodel/bundle/bundle_test.go @@ -0,0 +1,234 @@ +package bundle + +import ( + "encoding/json" + "errors" + "os" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/workflowmodel" +) + +var fixedNow = time.Date(2026, 11, 2, 12, 0, 0, 0, time.UTC) + +func techdocs(t *testing.T) *workflowmodel.Model { + t.Helper() + data, err := os.ReadFile("../testdata/models/techdocs.json") + if err != nil { + t.Fatal(err) + } + var m workflowmodel.Model + if err := json.Unmarshal(data, &m); err != nil { + t.Fatal(err) + } + return &m +} + +func TestExportRoundTrip(t *testing.T) { + m := techdocs(t) + b, err := Export(m, " andy ", fixedNow) + if err != nil { + t.Fatal(err) + } + if b.APIVersion != APIVersion || b.Kind != Kind || b.CreatedBy != "andy" || b.Tier != TierLocal { + t.Fatalf("envelope = %+v", b) + } + if b.Key() != "cncf-techdocs@1" { + t.Fatalf("key = %q", b.Key()) + } + if b.Hash != CanonicalHash(m) || len(b.Hash) != 64 { + t.Fatalf("hash = %q", b.Hash) + } + enc, err := Encode(b) + if err != nil { + t.Fatal(err) + } + // Canonical: no whitespace, sorted top-level keys. + if strings.Contains(string(enc), "\n") || !strings.HasPrefix(string(enc), `{"apiVersion":`) { + t.Fatalf("not canonical: %.80s", enc) + } + enc2, _ := Encode(b) + if string(enc) != string(enc2) { + t.Fatal("Encode not deterministic") + } + got, err := Decode(enc) + if err != nil { + t.Fatal(err) + } + if err := Verify(got); err != nil { + t.Fatalf("verify: %v", err) + } + if !got.CreatedAt.Equal(fixedNow) || got.Profile.Name != "cncf-techdocs" { + t.Fatalf("decoded = %+v", got) + } + // Key order in the source document does not change the hash. + reordered := `{"spec":` + mustJSON(t, m.Spec) + `,"metadata":` + mustJSON(t, m.Metadata) + `,"kind":"` + m.Kind + `","apiVersion":"` + m.APIVersion + `"}` + var m2 workflowmodel.Model + if err := json.Unmarshal([]byte(reordered), &m2); err != nil { + t.Fatal(err) + } + if CanonicalHash(&m2) != b.Hash { + t.Fatal("CanonicalHash depends on key order") + } +} + +func TestExportRefusals(t *testing.T) { + if _, err := Export(nil, "", fixedNow); !errors.Is(err, ErrMissingModel) { + t.Fatalf("nil model: %v", err) + } + m := techdocs(t) + m.Spec.Initial = "nope" + if _, err := Export(m, "", fixedNow); err == nil { + t.Fatal("expected invalid/non-compiling model to be refused") + } + m = techdocs(t) + m.Spec.Meta.Provenance = "mystery" + if _, err := Export(m, "", fixedNow); !errors.Is(err, ErrBadTier) { + t.Fatalf("bad tier: %v", err) + } + m = techdocs(t) + m.Spec.Meta.Provenance = workflowmodel.ProvenanceCurated + b, err := Export(m, "", fixedNow) + if err != nil || b.Tier != TierCurated { + t.Fatalf("curated: %v %+v", err, b) + } + // A model whose transitions fail to compile but which passes Validate. + m = techdocs(t) + m.Spec.Transitions = nil + m.Spec.Columns = nil + if _, err := Export(m, "", fixedNow); err == nil { + t.Log("stripped model still compiled; compile refusal is covered by Verify tests") + } +} + +func TestTamperedModel(t *testing.T) { + b, err := Export(techdocs(t), "andy", fixedNow) + if err != nil { + t.Fatal(err) + } + b.Model.Spec.Meta.Description = "tampered" + if err := Verify(b); !errors.Is(err, ErrHashMismatch) { + t.Fatalf("tampered model: %v", err) + } + // Re-hash to slip past the hash check with a model that no longer + // validates: Verify must still refuse. + b.Model.Spec.Initial = "nope" + b.Hash = CanonicalHash(b.Model) + if err := Verify(b); err == nil || errors.Is(err, ErrHashMismatch) { + t.Fatalf("invalid model with matching hash: %v", err) + } +} + +func TestTamperedProfile(t *testing.T) { + b, err := Export(techdocs(t), "andy", fixedNow) + if err != nil { + t.Fatal(err) + } + b.Profile.Columns = b.Profile.Columns[:1] + verr := Verify(b) + if !errors.Is(verr, ErrProfileMismatch) { + t.Fatalf("tampered profile: %v", verr) + } + if !strings.Contains(verr.Error(), "published profile does not match local compile") { + t.Fatalf("message = %q", verr.Error()) + } + // The model's own Profile.States is json:"-", so differences there are + // not part of the structural comparison. + b, _ = Export(techdocs(t), "andy", fixedNow) + b.Profile.States = nil + if err := Verify(b); err != nil { + t.Fatalf("States is excluded from comparison: %v", err) + } +} + +func TestEnvelopeErrors(t *testing.T) { + b, err := Export(techdocs(t), "andy", fixedNow) + if err != nil { + t.Fatal(err) + } + cases := map[string]struct { + mutate func(*Bundle) + want error + }{ + "bad version": {func(b *Bundle) { b.APIVersion = "v0" }, ErrBadVersion}, + "bad kind": {func(b *Bundle) { b.Kind = "Other" }, ErrBadKind}, + "missing profile": {func(b *Bundle) { b.Profile = nil }, ErrMissingProfile}, + "bad tier": {func(b *Bundle) { b.Tier = "gold" }, ErrBadTier}, + } + for name, tc := range cases { + c := *b + tc.mutate(&c) + if err := Verify(&c); !errors.Is(err, tc.want) { + t.Errorf("%s: Verify = %v, want %v", name, err, tc.want) + } + if tc.want == ErrBadTier { + continue + } + enc, _ := Encode(&c) + if _, err := Decode(enc); !errors.Is(err, tc.want) { + t.Errorf("%s: Decode = %v, want %v", name, err, tc.want) + } + } + if err := Verify(nil); !errors.Is(err, ErrMissingModel) { + t.Fatalf("nil: %v", err) + } + if _, err := Encode(nil); !errors.Is(err, ErrMissingModel) { + t.Fatalf("encode nil: %v", err) + } + c := *b + c.Kind = "" + if err := Verify(&c); err != nil { + t.Fatalf("empty kind is tolerated: %v", err) + } + enc, _ := Encode(&c) + d, err := Decode(enc) + if err != nil || d.Kind != Kind { + t.Fatalf("Decode should default kind: %v %q", err, d.Kind) + } + c = *b + c.Model = nil + enc, _ = Encode(&c) + if _, err := Decode(enc); !errors.Is(err, ErrMissingModel) { + t.Fatalf("missing model: %v", err) + } + if _, err := Decode([]byte(`{"apiVersion":"` + APIVersion + `","extra":1}`)); err == nil { + t.Fatal("unknown fields must be rejected") + } + if _, err := Decode([]byte("{")); err == nil { + t.Fatal("malformed JSON must be rejected") + } +} + +func TestHelpers(t *testing.T) { + if CanonicalHash(nil) != "" { + t.Fatal("nil hash") + } + var nilB *Bundle + if nilB.Key() != "" { + t.Fatal("nil key") + } + if short("") != "" || short("abc") != "abc" || short("0123456789abcdef") != "0123456789ab" { + t.Fatal("short") + } + if KnownTier(TierLocalPublished) != true || KnownTier("x") { + t.Fatal("KnownTier") + } + if _, err := canonicalJSON(make(chan int)); err == nil { + t.Fatal("unmarshalable value must error") + } + out, err := canonicalJSON(map[string]interface{}{"b": []interface{}{1, "x", nil}, "a": map[string]interface{}{"z": true, "y": 1.5}}) + if err != nil || string(out) != `{"a":{"y":1.5,"z":true},"b":[1,"x",null]}` { + t.Fatalf("canonical = %s %v", out, err) + } +} + +func mustJSON(t *testing.T, v interface{}) string { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return string(b) +} diff --git a/src/pkg/workflowmodel/ingest/corpus/corpus.go b/src/pkg/workflowmodel/ingest/corpus/corpus.go new file mode 100644 index 0000000000..a9139905f8 --- /dev/null +++ b/src/pkg/workflowmodel/ingest/corpus/corpus.go @@ -0,0 +1,90 @@ +// Package corpus embeds the CNCF workflow-survey corpus: the list of +// projects (and their GitHub repositories) whose CI/CD pipeline files were +// surveyed when designing the workflow model (#17620). The hub marketplace +// refresh seeds its `ingested-unreviewed` tier from this list, and +// cmd/workflow-survey falls back to it when a corpus directory carries no +// projects.json of its own. +package corpus + +import ( + _ "embed" + "encoding/json" + "sort" + "strings" + "sync" +) + +//go:embed corpus.json +var corpusJSON []byte + +// Project is one surveyed CNCF project. +type Project struct { + Name string `json:"name"` + Maturity string `json:"maturity"` + Category string `json:"category"` + Repos []string `json:"repos"` +} + +var ( + loadOnce sync.Once + loaded []Project + loadErr error +) + +func load() ([]Project, error) { + loadOnce.Do(func() { + var ps []Project + if err := json.Unmarshal(corpusJSON, &ps); err != nil { + loadErr = err + return + } + loaded = ps + }) + return loaded, loadErr +} + +// Projects returns a copy of the embedded project list, sorted by name. +func Projects() []Project { + ps, err := load() + if err != nil { + panic("corpus: embedded corpus.json is invalid: " + err.Error()) + } + out := make([]Project, len(ps)) + for i, p := range ps { + out[i] = p + out[i].Repos = append([]string(nil), p.Repos...) + } + return out +} + +// Repos returns every distinct `owner/repo` in the corpus, lower-cased and +// sorted, so callers iterate deterministically. +func Repos() []string { + seen := map[string]bool{} + var out []string + for _, p := range Projects() { + for _, r := range p.Repos { + r = strings.ToLower(strings.TrimSpace(r)) + if r == "" || seen[r] { + continue + } + seen[r] = true + out = append(out, r) + } + } + sort.Strings(out) + return out +} + +// Maturity maps lower-cased `owner/repo` to the CNCF maturity of the project +// that owns it (graduated/incubating/sandbox). Mirrors the shape consumed by +// cmd/workflow-survey's loadProjects. +func Maturity() map[string]string { + out := map[string]string{} + for _, p := range Projects() { + for _, r := range p.Repos { + out[strings.ToLower(r)] = p.Maturity + } + } + return out +} diff --git a/src/pkg/workflowmodel/ingest/corpus/corpus.json b/src/pkg/workflowmodel/ingest/corpus/corpus.json new file mode 100644 index 0000000000..cfe16ee3bf --- /dev/null +++ b/src/pkg/workflowmodel/ingest/corpus/corpus.json @@ -0,0 +1,2133 @@ +[ + { + "name": "Aeraki Mesh", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "aeraki-mesh/aeraki" + ] + }, + { + "name": "Agent Substrate", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "agent-substrate/substrate" + ] + }, + { + "name": "Agones", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "agones-dev/agones" + ] + }, + { + "name": "Akri", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "project-akri/akri" + ] + }, + { + "name": "Antrea", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "antrea-io/antrea" + ] + }, + { + "name": "Apicurio Registry", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "Apicurio/apicurio-registry" + ] + }, + { + "name": "Argo", + "maturity": "graduated", + "category": "App Definition and Development", + "repos": [ + "argoproj/argo-cd" + ] + }, + { + "name": "Armada", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "armadaproject/armada" + ] + }, + { + "name": "Artifact Hub", + "maturity": "incubating", + "category": "App Definition and Development", + "repos": [ + "artifacthub/hub" + ] + }, + { + "name": "Athenz", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "AthenZ/athenz" + ] + }, + { + "name": "Atlantis", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "runatlantis/atlantis" + ] + }, + { + "name": "Backstage", + "maturity": "incubating", + "category": "App Definition and Development", + "repos": [ + "backstage/backstage" + ] + }, + { + "name": "Bank-Vaults", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "bank-vaults/bank-vaults", + "bank-vaults/secrets-webhook", + "bank-vaults/vault-operator", + "bank-vaults/vault-sdk" + ] + }, + { + "name": "BFE", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "bfenetworks/bfe" + ] + }, + { + "name": "bootc", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "bootc-dev/bootc" + ] + }, + { + "name": "bpfman", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "bpfman/bpfman", + "bpfman/bpfman-operator" + ] + }, + { + "name": "Brigade", + "maturity": "archived", + "category": "App Definition and Development", + "repos": [ + "brigadecore/brigade" + ] + }, + { + "name": "Buildpacks", + "maturity": "graduated", + "category": "App Definition and Development", + "repos": [ + "buildpacks/pack" + ] + }, + { + "name": "Cadence Workflow", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "cadence-workflow/cadence" + ] + }, + { + "name": "CAIPE", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "caipe-io/agentic-apps", + "caipe-io/ai-platform-engineering", + "caipe-io/caipe-cli", + "caipe-io/governance" + ] + }, + { + "name": "Capsule", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "projectcapsule/capsule" + ] + }, + { + "name": "Carina", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "carina-io/carina" + ] + }, + { + "name": "Cartography", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "cartography-cncf/cartography" + ] + }, + { + "name": "Carvel", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "carvel-dev/ytt" + ] + }, + { + "name": "CDK for Kubernetes (CDK8s)", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "cdk8s-team/cdk8s" + ] + }, + { + "name": "Cedar", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "cedar-policy/cedar", + "cedar-policy/cedar-access-control-for-k8s" + ] + }, + { + "name": "cert-manager", + "maturity": "graduated", + "category": "Provisioning", + "repos": [ + "cert-manager/cert-manager" + ] + }, + { + "name": "Chaos Mesh", + "maturity": "incubating", + "category": "Observability and Analysis", + "repos": [ + "chaos-mesh/chaos-mesh" + ] + }, + { + "name": "Chaosblade", + "maturity": "sandbox", + "category": "Observability and Analysis", + "repos": [ + "chaosblade-io/chaosblade" + ] + }, + { + "name": "Cilium", + "maturity": "graduated", + "category": "Runtime", + "repos": [ + "cilium/cilium" + ] + }, + { + "name": "Cloud Custodian", + "maturity": "incubating", + "category": "Provisioning", + "repos": [ + "cloud-custodian/cloud-custodian" + ] + }, + { + "name": "CloudEvents", + "maturity": "graduated", + "category": "App Definition and Development", + "repos": [ + "cloudevents/spec" + ] + }, + { + "name": "CloudNativePG", + "maturity": "incubating", + "category": "App Definition and Development", + "repos": [ + "cloudnative-pg/cloudnative-pg" + ] + }, + { + "name": "Clusternet", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "clusternet/clusternet" + ] + }, + { + "name": "Clusterpedia", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "clusterpedia-io/clusterpedia" + ] + }, + { + "name": "CNI-Genie", + "maturity": "archived", + "category": "Runtime", + "repos": [ + "cni-genie/CNI-Genie" + ] + }, + { + "name": "CoHDI", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [] + }, + { + "name": "composefs", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "containers/composefs" + ] + }, + { + "name": "Confidential Containers", + "maturity": "incubating", + "category": "Provisioning", + "repos": [ + "confidential-containers/confidential-containers" + ] + }, + { + "name": "Connect RPC", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "connectrpc/connect-go" + ] + }, + { + "name": "Container Network Interface (CNI)", + "maturity": "incubating", + "category": "Runtime", + "repos": [ + "containernetworking/cni" + ] + }, + { + "name": "container2wasm", + "maturity": "sandbox", + "category": "Wasm", + "repos": [ + "container2wasm/container2wasm" + ] + }, + { + "name": "containerd", + "maturity": "graduated", + "category": "Runtime", + "repos": [ + "containerd/containerd" + ] + }, + { + "name": "ContainerSSH", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "containerssh/containerssh" + ] + }, + { + "name": "Contour", + "maturity": "incubating", + "category": "Orchestration & Management", + "repos": [ + "projectcontour/contour" + ] + }, + { + "name": "Copa", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "project-copacetic/copacetic" + ] + }, + { + "name": "CoreDNS", + "maturity": "graduated", + "category": "Orchestration & Management", + "repos": [ + "coredns/coredns" + ] + }, + { + "name": "Cortex", + "maturity": "incubating", + "category": "Observability and Analysis", + "repos": [ + "cortexproject/cortex" + ] + }, + { + "name": "Cozystack", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "cozystack/cozystack" + ] + }, + { + "name": "CRI-O", + "maturity": "graduated", + "category": "Runtime", + "repos": [ + "cri-o/cri-o" + ] + }, + { + "name": "Crossplane", + "maturity": "graduated", + "category": "Orchestration & Management", + "repos": [ + "crossplane/crossplane" + ] + }, + { + "name": "CubeFS", + "maturity": "graduated", + "category": "Runtime", + "repos": [ + "cubeFS/cubefs" + ] + }, + { + "name": "Curiefense", + "maturity": "archived", + "category": "Provisioning", + "repos": [] + }, + { + "name": "Curve", + "maturity": "archived", + "category": "Runtime", + "repos": [ + "opencurve/curve" + ] + }, + { + "name": "Curvine", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "CurvineIO/curvine" + ] + }, + { + "name": "Dalec", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "project-dalec/dalec" + ] + }, + { + "name": "Dapr", + "maturity": "graduated", + "category": "App Definition and Development", + "repos": [ + "dapr/dapr" + ] + }, + { + "name": "Devfile", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "devfile/api" + ] + }, + { + "name": "DevSpace", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "devspace-sh/devspace" + ] + }, + { + "name": "DevStream", + "maturity": "archived", + "category": "Provisioning", + "repos": [ + "devstream-io/devstream" + ] + }, + { + "name": "Dex", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "dexidp/dex" + ] + }, + { + "name": "Distribution", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "distribution/distribution" + ] + }, + { + "name": "Dragonfly", + "maturity": "graduated", + "category": "Provisioning", + "repos": [ + "dragonflyoss/dragonfly" + ] + }, + { + "name": "Drasi", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "drasi-project/drasi-platform" + ] + }, + { + "name": "Easegress", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "easegress-io/easegress" + ] + }, + { + "name": "Emissary-Ingress", + "maturity": "incubating", + "category": "Orchestration & Management", + "repos": [ + "emissary-ingress/emissary" + ] + }, + { + "name": "Envoy", + "maturity": "graduated", + "category": "Orchestration & Management", + "repos": [ + "envoyproxy/envoy" + ] + }, + { + "name": "Eraser", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "eraser-dev/eraser" + ] + }, + { + "name": "etcd", + "maturity": "graduated", + "category": "Orchestration & Management", + "repos": [ + "etcd-io/etcd" + ] + }, + { + "name": "external-secrets", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "external-secrets/external-secrets" + ] + }, + { + "name": "FabEdge", + "maturity": "archived", + "category": "Runtime", + "repos": [ + "FabEdge/fabedge" + ] + }, + { + "name": "Falco", + "maturity": "graduated", + "category": "Provisioning", + "repos": [ + "falcosecurity/falco" + ] + }, + { + "name": "Flatcar Container Linux", + "maturity": "incubating", + "category": "Platform", + "repos": [ + "flatcar/Flatcar" + ] + }, + { + "name": "Fluentd", + "maturity": "graduated", + "category": "Observability and Analysis", + "repos": [ + "fluent/fluentd" + ] + }, + { + "name": "Fluid", + "maturity": "incubating", + "category": "Orchestration & Management", + "repos": [ + "fluid-cloudnative/fluid" + ] + }, + { + "name": "Flux", + "maturity": "graduated", + "category": "App Definition and Development", + "repos": [ + "fluxcd/flux2" + ] + }, + { + "name": "Fonio", + "maturity": "archived", + "category": "Observability and Analysis", + "repos": [ + "foniod/foniod" + ] + }, + { + "name": "gRPC", + "maturity": "incubating", + "category": "Orchestration & Management", + "repos": [ + "grpc/grpc" + ] + }, + { + "name": "HAMi", + "maturity": "incubating", + "category": "Orchestration & Management", + "repos": [ + "Project-HAMi/HAMi" + ] + }, + { + "name": "Harbor", + "maturity": "graduated", + "category": "Provisioning", + "repos": [ + "goharbor/harbor" + ] + }, + { + "name": "Headlamp", + "maturity": "sandbox", + "category": "Observability and Analysis", + "repos": [ + "kubernetes-sigs/headlamp" + ] + }, + { + "name": "Helm", + "maturity": "graduated", + "category": "App Definition and Development", + "repos": [ + "helm/helm" + ] + }, + { + "name": "Hexa", + "maturity": "archived", + "category": "Provisioning", + "repos": [ + "hexa-org/policy-orchestrator" + ] + }, + { + "name": "Higress", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "higress-group/higress" + ] + }, + { + "name": "Hive Commons", + "maturity": "sandbox", + "category": "AI Agent", + "repos": [ + "hivecommons/hive" + ] + }, + { + "name": "HolmesGPT", + "maturity": "sandbox", + "category": "Observability and Analysis", + "repos": [ + "HolmesGPT/holmesgpt" + ] + }, + { + "name": "HwameiStor", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "hwameistor/hwameistor" + ] + }, + { + "name": "Hyperlight", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "hyperlight-dev/hyperlight" + ] + }, + { + "name": "in-toto", + "maturity": "graduated", + "category": "Provisioning", + "repos": [ + "in-toto/in-toto" + ] + }, + { + "name": "Inclavare Containers", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "inclavare-containers/inclavare-containers" + ] + }, + { + "name": "Inspektor Gadget", + "maturity": "sandbox", + "category": "Observability and Analysis", + "repos": [ + "inspektor-gadget/inspektor-gadget" + ] + }, + { + "name": "Interlink", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "interlink-hq/interLink" + ] + }, + { + "name": "Istio", + "maturity": "graduated", + "category": "Orchestration & Management", + "repos": [ + "istio/istio" + ] + }, + { + "name": "Jaeger", + "maturity": "graduated", + "category": "Observability and Analysis", + "repos": [ + "jaegertracing/jaeger" + ] + }, + { + "name": "k0s", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "k0sproject/k0s" + ] + }, + { + "name": "k3s", + "maturity": "sandbox", + "category": "Platform", + "repos": [ + "k3s-io/k3s" + ] + }, + { + "name": "k8gb", + "maturity": "incubating", + "category": "Orchestration & Management", + "repos": [ + "k8gb-io/k8gb" + ] + }, + { + "name": "K8sGPT", + "maturity": "sandbox", + "category": "Observability and Analysis", + "repos": [ + "k8sgpt-ai/k8sgpt" + ] + }, + { + "name": "K8up", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "k8up-io/k8up" + ] + }, + { + "name": "kagent", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "kagent-dev/kagent" + ] + }, + { + "name": "KAI Scheduler", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "kai-scheduler/KAI-Scheduler" + ] + }, + { + "name": "Kairos", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "kairos-io/kairos" + ] + }, + { + "name": "KAITO", + "maturity": "sandbox", + "category": "Inference", + "repos": [ + "kaito-project/kaito" + ] + }, + { + "name": "Kanister", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "kanisterio/kanister" + ] + }, + { + "name": "Karmada", + "maturity": "graduated", + "category": "Orchestration & Management", + "repos": [ + "karmada-io/karmada" + ] + }, + { + "name": "kbind", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "kbind-dev/kbind" + ] + }, + { + "name": "KCL", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "kcl-lang/kcl" + ] + }, + { + "name": "kcp", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "kcp-dev/kcp" + ] + }, + { + "name": "KEDA", + "maturity": "graduated", + "category": "Orchestration & Management", + "repos": [ + "kedacore/keda" + ] + }, + { + "name": "Kepler", + "maturity": "sandbox", + "category": "Observability and Analysis", + "repos": [ + "sustainable-computing-io/kepler" + ] + }, + { + "name": "Keptn", + "maturity": "archived", + "category": "App Definition and Development", + "repos": [ + "keptn/lifecycle-toolkit" + ] + }, + { + "name": "Keycloak", + "maturity": "incubating", + "category": "Provisioning", + "repos": [ + "keycloak/keycloak" + ] + }, + { + "name": "Keylime", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "keylime/keylime" + ] + }, + { + "name": "Kgateway", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "kgateway-dev/kgateway" + ] + }, + { + "name": "KitOps", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "kitops-ml/kitops" + ] + }, + { + "name": "Kmesh", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "kmesh-net/kmesh" + ] + }, + { + "name": "Knative", + "maturity": "graduated", + "category": "Orchestration & Management", + "repos": [ + "knative/serving" + ] + }, + { + "name": "ko", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "ko-build/ko" + ] + }, + { + "name": "Konveyor", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "konveyor/operator" + ] + }, + { + "name": "Koordinator", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "koordinator-sh/koordinator" + ] + }, + { + "name": "kpt", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "kptdev/kpt" + ] + }, + { + "name": "Krator", + "maturity": "archived", + "category": "App Definition and Development", + "repos": [ + "krator-rs/krator" + ] + }, + { + "name": "Krkn", + "maturity": "sandbox", + "category": "Observability and Analysis", + "repos": [ + "krkn-chaos/krkn" + ] + }, + { + "name": "Krustlet", + "maturity": "archived", + "category": "Runtime", + "repos": [ + "krustlet/krustlet" + ] + }, + { + "name": "KServe", + "maturity": "incubating", + "category": "Inference", + "repos": [ + "kserve/kserve" + ] + }, + { + "name": "Kuadrant", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "kuadrant/kuadrant-operator" + ] + }, + { + "name": "Kuasar", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "kuasar-io/kuasar" + ] + }, + { + "name": "Kube-burner", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "kube-burner/kube-burner" + ] + }, + { + "name": "Kube-OVN", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "kubeovn/kube-ovn" + ] + }, + { + "name": "kube-rs", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "kube-rs/kube" + ] + }, + { + "name": "kube-vip", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "kube-vip/kube-vip" + ] + }, + { + "name": "Kubean", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "kubean-io/kubean" + ] + }, + { + "name": "KubeArmor", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "kubearmor/kubearmor" + ] + }, + { + "name": "KubeClipper", + "maturity": "sandbox", + "category": "Platform", + "repos": [ + "kubeclipper/kubeclipper" + ] + }, + { + "name": "KubeDL", + "maturity": "archived", + "category": "Provisioning", + "repos": [ + "kubedl-io/kubedl" + ] + }, + { + "name": "KubeEdge", + "maturity": "graduated", + "category": "Provisioning", + "repos": [ + "kubeedge/kubeedge" + ] + }, + { + "name": "KubeElasti", + "maturity": "sandbox", + "category": "Serverless", + "repos": [ + "KubeElasti/KubeElasti" + ] + }, + { + "name": "KubeFleet", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "kubefleet-dev/kubefleet" + ] + }, + { + "name": "Kubeflow", + "maturity": "graduated", + "category": "Orchestration & Management", + "repos": [ + "kubeflow/kubeflow" + ] + }, + { + "name": "KubeRay", + "maturity": "sandbox", + "category": "AI Native Infra", + "repos": [ + "ray-project/kuberay" + ] + }, + { + "name": "Kuberhealthy", + "maturity": "sandbox", + "category": "Observability and Analysis", + "repos": [ + "kuberhealthy/kuberhealthy" + ] + }, + { + "name": "Kubernetes", + "maturity": "graduated", + "category": "Orchestration & Management", + "repos": [ + "kubernetes/kubernetes" + ] + }, + { + "name": "Kubescape", + "maturity": "incubating", + "category": "Provisioning", + "repos": [ + "kubescape/kubescape" + ] + }, + { + "name": "KubeSlice", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "kubeslice/kubeslice" + ] + }, + { + "name": "KubeStellar", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "kubestellar/kubestellar" + ] + }, + { + "name": "KubeVela", + "maturity": "incubating", + "category": "App Definition and Development", + "repos": [ + "kubevela/kubevela" + ] + }, + { + "name": "KubeVirt", + "maturity": "graduated", + "category": "App Definition and Development", + "repos": [ + "kubevirt/kubevirt" + ] + }, + { + "name": "Kubewarden", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "kubewarden/adm-controller", + "kubewarden/community", + "kubewarden/helm-charts", + "kubewarden/policies", + "kubewarden/sbomscanner" + ] + }, + { + "name": "KUDO", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "kudobuilder/kudo" + ] + }, + { + "name": "Kuma", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "kumahq/kuma" + ] + }, + { + "name": "Kured", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "kubereboot/kured" + ] + }, + { + "name": "KusionStack", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "KusionStack/kusion" + ] + }, + { + "name": "Kyverno", + "maturity": "graduated", + "category": "Provisioning", + "repos": [ + "kyverno/kyverno" + ] + }, + { + "name": "Lima", + "maturity": "incubating", + "category": "Runtime", + "repos": [ + "lima-vm/lima" + ] + }, + { + "name": "Linkerd", + "maturity": "graduated", + "category": "Orchestration & Management", + "repos": [ + "linkerd/linkerd2" + ] + }, + { + "name": "Litmus", + "maturity": "incubating", + "category": "Observability and Analysis", + "repos": [ + "litmuschaos/litmus" + ] + }, + { + "name": "llm-d", + "maturity": "sandbox", + "category": "Inference", + "repos": [ + "llm-d/llm-d" + ] + }, + { + "name": "Logging Operator (Kube Logging)", + "maturity": "sandbox", + "category": "Observability and Analysis", + "repos": [ + "kube-logging/logging-operator" + ] + }, + { + "name": "Longhorn", + "maturity": "incubating", + "category": "Runtime", + "repos": [ + "longhorn/longhorn" + ] + }, + { + "name": "LoxiLB", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "loxilb-io/loxilb" + ] + }, + { + "name": "MatrixHub", + "maturity": "sandbox", + "category": "AI Native Infra", + "repos": [ + "matrixhub-ai/matrixhub" + ] + }, + { + "name": "Merbridge", + "maturity": "archived", + "category": "Orchestration & Management", + "repos": [ + "merbridge/merbridge" + ] + }, + { + "name": "Meshery", + "maturity": "incubating", + "category": "Provisioning", + "repos": [ + "meshery/meshery" + ] + }, + { + "name": "metal3-io", + "maturity": "incubating", + "category": "Provisioning", + "repos": [ + "metal3-io/baremetal-operator" + ] + }, + { + "name": "MetalLB", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "metallb/metallb" + ] + }, + { + "name": "Microcks", + "maturity": "incubating", + "category": "App Definition and Development", + "repos": [ + "microcks/microcks" + ] + }, + { + "name": "ModelPack", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "modelpack/model-spec" + ] + }, + { + "name": "NATS", + "maturity": "graduated", + "category": "App Definition and Development", + "repos": [ + "nats-io/jetstream-gh-action", + "nats-io/k8s", + "nats-io/nack", + "nats-io/nats-architecture-and-design", + "nats-io/nats-server", + "nats-io/nats-surveyor", + "nats-io/nats-top", + "nats-io/nats.c", + "nats-io/nats.deno", + "nats-io/nats.docs", + "nats-io/nats.go", + "nats-io/nats.java", + "nats-io/nats.js", + "nats-io/nats.net", + "nats-io/nats.py", + "nats-io/nats.rs", + "nats-io/natscli", + "nats-io/terraform-provider-jetstream" + ] + }, + { + "name": "Network Service Mesh", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "networkservicemesh/api" + ] + }, + { + "name": "NMstate", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "nmstate/nmstate" + ] + }, + { + "name": "Nocalhost", + "maturity": "archived", + "category": "App Definition and Development", + "repos": [ + "nocalhost/nocalhost" + ] + }, + { + "name": "Notary Project", + "maturity": "incubating", + "category": "Provisioning", + "repos": [ + "notaryproject/notation" + ] + }, + { + "name": "OAuth2 Proxy", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "oauth2-proxy/manifests", + "oauth2-proxy/oauth2-proxy" + ] + }, + { + "name": "Open Cluster Management", + "maturity": "incubating", + "category": "Orchestration & Management", + "repos": [ + "open-cluster-management-io/ocm" + ] + }, + { + "name": "Open Policy Agent (OPA)", + "maturity": "graduated", + "category": "Provisioning", + "repos": [ + "open-policy-agent/opa" + ] + }, + { + "name": "Open Policy Containers", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "opcr-io/policy" + ] + }, + { + "name": "Open Service Mesh", + "maturity": "archived", + "category": "Orchestration & Management", + "repos": [ + "openservicemesh/osm" + ] + }, + { + "name": "Open Workflow Specification", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "open-workflow-specification/specification" + ] + }, + { + "name": "OpenChoreo", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "openchoreo/openchoreo" + ] + }, + { + "name": "OpenCost", + "maturity": "incubating", + "category": "Observability and Analysis", + "repos": [ + "opencost/opencost" + ] + }, + { + "name": "OpenEBS", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "openebs/openebs" + ] + }, + { + "name": "OpenELB", + "maturity": "archived", + "category": "Orchestration & Management", + "repos": [ + "openelb/openelb" + ] + }, + { + "name": "OpenEverest", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "openeverest/openeverest" + ] + }, + { + "name": "OpenFeature", + "maturity": "incubating", + "category": "Observability and Analysis", + "repos": [ + "open-feature/spec" + ] + }, + { + "name": "OpenFGA", + "maturity": "incubating", + "category": "Provisioning", + "repos": [ + "openfga/openfga" + ] + }, + { + "name": "OpenFunction", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "OpenFunction/OpenFunction" + ] + }, + { + "name": "openGemini", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "openGemini/openGemini" + ] + }, + { + "name": "OpenGitOps", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "open-gitops/project" + ] + }, + { + "name": "OpenKruise", + "maturity": "incubating", + "category": "App Definition and Development", + "repos": [ + "openkruise/kruise" + ] + }, + { + "name": "OpenMetrics", + "maturity": "archived", + "category": "Observability and Analysis", + "repos": [ + "OpenObservability/OpenMetrics" + ] + }, + { + "name": "OpenShell", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "NVIDIA/OpenShell" + ] + }, + { + "name": "OpenTelemetry", + "maturity": "graduated", + "category": "Observability and Analysis", + "repos": [ + "open-telemetry/community" + ] + }, + { + "name": "OpenTofu", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "opentofu/opentofu" + ] + }, + { + "name": "OpenTracing", + "maturity": "archived", + "category": "Observability and Analysis", + "repos": [ + "opentracing/opentracing-go" + ] + }, + { + "name": "OpenYurt", + "maturity": "incubating", + "category": "Provisioning", + "repos": [ + "openyurtio/openyurt" + ] + }, + { + "name": "Operator Framework", + "maturity": "incubating", + "category": "App Definition and Development", + "repos": [ + "operator-framework/operator-sdk" + ] + }, + { + "name": "ORAS", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "oras-project/oras" + ] + }, + { + "name": "OSCAL-COMPASS", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "oscal-compass/compliance-to-policy", + "oscal-compass/compliance-trestle", + "oscal-compass/compliance-trestle-agile-authoring", + "oscal-compass/compliance-trestle-demos" + ] + }, + { + "name": "OVN-Kubernetes", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "ovn-kubernetes/ovn-kubernetes" + ] + }, + { + "name": "Oxia", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "oxia-db/oxia", + "oxia-db/oxia-client-java" + ] + }, + { + "name": "Paralus", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "paralus/paralus" + ] + }, + { + "name": "Parsec", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "parallaxsecond/parsec" + ] + }, + { + "name": "Perses", + "maturity": "sandbox", + "category": "Observability and Analysis", + "repos": [ + "perses/perses" + ] + }, + { + "name": "PipeCD", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "pipe-cd/pipecd" + ] + }, + { + "name": "Piraeus Datastore", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "piraeusdatastore/piraeus-operator" + ] + }, + { + "name": "Pixie", + "maturity": "sandbox", + "category": "Observability and Analysis", + "repos": [ + "pixie-io/pixie" + ] + }, + { + "name": "Podman Container Tools", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "podman-container-tools/podman" + ] + }, + { + "name": "Podman Desktop", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "podman-desktop/podman-desktop" + ] + }, + { + "name": "Porter", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "getporter/porter" + ] + }, + { + "name": "Pravega", + "maturity": "archived", + "category": "App Definition and Development", + "repos": [ + "pravega/pravega" + ] + }, + { + "name": "Praxis", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "praxis-proxy/praxis" + ] + }, + { + "name": "Prometheus", + "maturity": "graduated", + "category": "Observability and Analysis", + "repos": [ + "prometheus/prometheus" + ] + }, + { + "name": "Radius", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "radius-project/radius" + ] + }, + { + "name": "Ratify", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "ratify-project/ratify" + ] + }, + { + "name": "rkt", + "maturity": "archived", + "category": "Runtime", + "repos": [ + "rkt/rkt" + ] + }, + { + "name": "Rook", + "maturity": "graduated", + "category": "Runtime", + "repos": [ + "rook/rook" + ] + }, + { + "name": "Runme Notebooks", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "runmedev/runme" + ] + }, + { + "name": "SchemaHero", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "schemahero/schemahero" + ] + }, + { + "name": "Score", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "score-spec/spec" + ] + }, + { + "name": "sealer", + "maturity": "archived", + "category": "App Definition and Development", + "repos": [ + "sealerio/sealer" + ] + }, + { + "name": "Sermant", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "sermant-io/Sermant" + ] + }, + { + "name": "Serverless Devs", + "maturity": "sandbox", + "category": "Orchestration & Management", + "repos": [ + "serverless-devs/serverless-devs" + ] + }, + { + "name": "Service Mesh Interface (SMI)", + "maturity": "archived", + "category": "Orchestration & Management", + "repos": [ + "servicemeshinterface/smi-spec" + ] + }, + { + "name": "Service Mesh Performance", + "maturity": "archived", + "category": "Orchestration & Management", + "repos": [ + "service-mesh-performance/service-mesh-performance" + ] + }, + { + "name": "Shipwright", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "shipwright-io/build" + ] + }, + { + "name": "Skooner", + "maturity": "archived", + "category": "Observability and Analysis", + "repos": [ + "skooner-k8s/skooner" + ] + }, + { + "name": "SlimFaaS", + "maturity": "sandbox", + "category": "Serverless", + "repos": [ + "SlimPlanet/SlimFaas" + ] + }, + { + "name": "SlimToolkit", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "slimtoolkit/slim" + ] + }, + { + "name": "SOPS", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "getsops/sops" + ] + }, + { + "name": "Spiderpool", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "spidernet-io/spiderpool" + ] + }, + { + "name": "SPIFFE", + "maturity": "graduated", + "category": "Provisioning", + "repos": [ + "spiffe/spiffe" + ] + }, + { + "name": "Spin", + "maturity": "sandbox", + "category": "Wasm", + "repos": [ + "spinframework/spin" + ] + }, + { + "name": "SpinKube", + "maturity": "sandbox", + "category": "Wasm", + "repos": [ + "spinframework/spin-operator" + ] + }, + { + "name": "SPIRE", + "maturity": "graduated", + "category": "Provisioning", + "repos": [ + "spiffe/spire" + ] + }, + { + "name": "Stacker", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "project-stacker/stacker" + ] + }, + { + "name": "Strimzi", + "maturity": "incubating", + "category": "App Definition and Development", + "repos": [ + "strimzi/strimzi-kafka-operator" + ] + }, + { + "name": "Submariner", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "submariner-io/submariner" + ] + }, + { + "name": "SuperEdge", + "maturity": "archived", + "category": "Provisioning", + "repos": [ + "superedge/superedge" + ] + }, + { + "name": "Tekton", + "maturity": "incubating", + "category": "App Definition and Development", + "repos": [ + "tektoncd/pipeline" + ] + }, + { + "name": "Telepresence", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "telepresenceio/telepresence" + ] + }, + { + "name": "Teller", + "maturity": "archived", + "category": "Provisioning", + "repos": [ + "tellerops/teller" + ] + }, + { + "name": "Thanos", + "maturity": "incubating", + "category": "Observability and Analysis", + "repos": [ + "thanos-io/thanos" + ] + }, + { + "name": "The Update Framework (TUF)", + "maturity": "graduated", + "category": "Provisioning", + "repos": [ + "theupdateframework/python-tuf" + ] + }, + { + "name": "TiKV", + "maturity": "graduated", + "category": "App Definition and Development", + "repos": [ + "tikv/tikv" + ] + }, + { + "name": "Tinkerbell", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "tinkerbell/tinkerbell" + ] + }, + { + "name": "Tokenetes", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "tokenetes/tokenetes" + ] + }, + { + "name": "Tremor", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "tremor-rs/tremor-runtime" + ] + }, + { + "name": "Trickster", + "maturity": "sandbox", + "category": "Observability and Analysis", + "repos": [ + "trickstercache/trickster" + ] + }, + { + "name": "urunc", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "urunc-dev/urunc" + ] + }, + { + "name": "Velero", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "velero-io/velero" + ] + }, + { + "name": "Vineyard", + "maturity": "archived", + "category": "Runtime", + "repos": [ + "v6d-io/v6d" + ] + }, + { + "name": "Virtual Kubelet", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "virtual-kubelet/virtual-kubelet" + ] + }, + { + "name": "Visual Studio Code Kubernetes Tools", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "vscode-kubernetes-tools/vscode-kubernetes-tools" + ] + }, + { + "name": "Vitess", + "maturity": "graduated", + "category": "App Definition and Development", + "repos": [ + "vitessio/arewefastyet", + "vitessio/enhancements", + "vitessio/vitess", + "vitessio/website" + ] + }, + { + "name": "Volcano", + "maturity": "incubating", + "category": "Orchestration & Management", + "repos": [ + "volcano-sh/volcano" + ] + }, + { + "name": "wasmCloud", + "maturity": "incubating", + "category": "Orchestration & Management", + "repos": [ + "wasmCloud/wasmCloud" + ] + }, + { + "name": "WasmEdge Runtime", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "WasmEdge/WasmEdge" + ] + }, + { + "name": "werf", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "werf/werf" + ] + }, + { + "name": "Xline", + "maturity": "archived", + "category": "Orchestration & Management", + "repos": [ + "xline-kv/Xline" + ] + }, + { + "name": "xRegistry", + "maturity": "sandbox", + "category": "App Definition and Development", + "repos": [ + "xregistry/server", + "xregistry/spec", + "xregistry/xregistry.github.io" + ] + }, + { + "name": "youki", + "maturity": "sandbox", + "category": "Runtime", + "repos": [ + "youki-dev/youki" + ] + }, + { + "name": "zot", + "maturity": "sandbox", + "category": "Provisioning", + "repos": [ + "project-zot/zot" + ] + } +] diff --git a/src/pkg/workflowmodel/ingest/corpus/corpus_test.go b/src/pkg/workflowmodel/ingest/corpus/corpus_test.go new file mode 100644 index 0000000000..2892626047 --- /dev/null +++ b/src/pkg/workflowmodel/ingest/corpus/corpus_test.go @@ -0,0 +1,60 @@ +package corpus + +import ( + "sort" + "strings" + "sync" + "testing" +) + +func TestProjectsAndRepos(t *testing.T) { + ps := Projects() + if len(ps) < 200 { + t.Fatalf("expected the full CNCF corpus, got %d projects", len(ps)) + } + if !sort.SliceIsSorted(ps, func(i, j int) bool { + return strings.ToLower(ps[i].Name) < strings.ToLower(ps[j].Name) + }) { + t.Fatal("projects not sorted by name") + } + repos := Repos() + if !sort.StringsAreSorted(repos) { + t.Fatal("repos not sorted") + } + seen := map[string]bool{} + for _, r := range repos { + if r != strings.ToLower(r) || !strings.Contains(r, "/") { + t.Fatalf("bad repo entry %q", r) + } + if seen[r] { + t.Fatalf("duplicate repo %q", r) + } + seen[r] = true + } + if !seen["project-akri/akri"] { + t.Fatal("expected project-akri/akri in corpus") + } + m := Maturity() + if m["project-akri/akri"] != "sandbox" { + t.Fatalf("maturity = %q", m["project-akri/akri"]) + } + // Projects() hands out copies. + ps[0].Repos[0] = "tampered" + if Projects()[0].Repos[0] == "tampered" { + t.Fatal("Projects leaked internal slice") + } +} + +func TestLoadInvalid(t *testing.T) { + old := corpusJSON + t.Cleanup(func() { corpusJSON = old; loadOnce = sync.Once{}; loaded, loadErr = nil, nil }) + loadOnce = sync.Once{} + loaded, loadErr = nil, nil + corpusJSON = []byte("{not json") + defer func() { + if recover() == nil { + t.Fatal("expected panic on invalid embedded corpus") + } + }() + Projects() +} From fd6f08b9b204a42b88debe847da8bb54df12a354 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 11 Nov 2026 09:31:10 -0700 Subject: [PATCH 2/9] =?UTF-8?q?=E2=9C=A8=20config:=20workflow.marketplace?= =?UTF-8?q?=20block=20(enabled,=20url,=20refresh=5Finterval)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --- src/pkg/config/config.go | 47 +++++++++++++++++++ .../config/testdata/agent_runtime_golden.json | 3 +- src/pkg/config/workflow_marketplace_test.go | 35 ++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 src/pkg/config/workflow_marketplace_test.go diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 810c5256dc..32da9850e6 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -296,6 +296,9 @@ type WorkflowConfig struct { // workflow_label_removed. nil means true; false leaves every label in // place and logs what would have gone. LabelGC *bool `yaml:"label_gc,omitempty" json:"label_gc,omitempty"` + // Marketplace configures the spoke's view of the hub workflow catalogue + // (hivecommons/hive#17620 phase 7). + Marketplace MarketplaceConfig `yaml:"marketplace,omitempty" json:"marketplace,omitempty"` } // LabelGCEnabled is the effective workflow.label_gc (default true). @@ -303,6 +306,50 @@ func (c WorkflowConfig) LabelGCEnabled() bool { return c.LabelGC == nil || *c.LabelGC } +// MarketplaceConfig is the `workflow.marketplace:` block. The spoke proxies +// the hub catalogue index, imports bundles from it and publishes library +// entries to it. +type MarketplaceConfig struct { + // Enabled defaults to true; set false to hide the Marketplace view and + // return 404 from the proxy routes. + Enabled *bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // URL is the hub base URL serving /api/marketplace/*. Empty means the + // configured hub.url (which itself defaults to the public hub). + URL string `yaml:"url,omitempty" json:"url,omitempty"` + // RefreshInterval bounds how long the spoke serves a cached index + // before revalidating with the hub (If-None-Match). Zero means the + // default of 15 minutes. + RefreshInterval time.Duration `yaml:"refresh_interval,omitempty" json:"refresh_interval,omitempty"` +} + +// DefaultMarketplaceRefreshInterval is the spoke-side index cache TTL. +const DefaultMarketplaceRefreshInterval = 15 * time.Minute + +// MarketplaceEnabled reports the effective workflow.marketplace.enabled. +func (m MarketplaceConfig) MarketplaceEnabled() bool { + return m.Enabled == nil || *m.Enabled +} + +// EffectiveRefreshInterval returns refresh_interval or its default. +func (m MarketplaceConfig) EffectiveRefreshInterval() time.Duration { + if m.RefreshInterval <= 0 { + return DefaultMarketplaceRefreshInterval + } + return m.RefreshInterval +} + +// MarketplaceURL resolves the hub base URL for marketplace calls: the +// explicit workflow.marketplace.url, else hub.url. +func (c *Config) MarketplaceURL() string { + if c == nil { + return "" + } + if u := strings.TrimSpace(c.Workflow.Marketplace.URL); u != "" { + return strings.TrimRight(u, "/") + } + return strings.TrimRight(strings.TrimSpace(c.Hub.URL), "/") +} + // UnmarshalYAML accepts the operator-facing agents.github_api_hourly_cap // scalar alongside the existing agents. map entries. The Agents field // remains a plain map everywhere else in the codebase. diff --git a/src/pkg/config/testdata/agent_runtime_golden.json b/src/pkg/config/testdata/agent_runtime_golden.json index 891512459a..a70784ec8f 100644 --- a/src/pkg/config/testdata/agent_runtime_golden.json +++ b/src/pkg/config/testdata/agent_runtime_golden.json @@ -670,7 +670,8 @@ "dedupe": {}, "contribute": {}, "workflow": { - "enabled": false + "enabled": false, + "marketplace": {} }, "HiveID": "", "acmm_level": null, diff --git a/src/pkg/config/workflow_marketplace_test.go b/src/pkg/config/workflow_marketplace_test.go new file mode 100644 index 0000000000..c810fb1cff --- /dev/null +++ b/src/pkg/config/workflow_marketplace_test.go @@ -0,0 +1,35 @@ +package config + +import ( + "testing" + "time" +) + +func TestMarketplaceConfigDefaults(t *testing.T) { + var m MarketplaceConfig + if !m.MarketplaceEnabled() { + t.Fatal("marketplace should default to enabled") + } + if m.EffectiveRefreshInterval() != DefaultMarketplaceRefreshInterval { + t.Fatalf("interval = %v", m.EffectiveRefreshInterval()) + } + off := false + m = MarketplaceConfig{Enabled: &off, RefreshInterval: time.Minute} + if m.MarketplaceEnabled() || m.EffectiveRefreshInterval() != time.Minute { + t.Fatalf("explicit values ignored: %+v", m) + } + + var nilCfg *Config + if nilCfg.MarketplaceURL() != "" { + t.Fatal("nil config URL") + } + cfg := &Config{} + cfg.Hub.URL = "https://hub.example/" + if got := cfg.MarketplaceURL(); got != "https://hub.example" { + t.Fatalf("falls back to hub.url, got %q", got) + } + cfg.Workflow.Marketplace.URL = " https://mkt.example// " + if got := cfg.MarketplaceURL(); got != "https://mkt.example" { + t.Fatalf("explicit url, got %q", got) + } +} From 9ca93309b50c89cb6408933ec680a0042880abb4 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 11 Nov 2026 09:38:56 -0700 Subject: [PATCH 3/9] =?UTF-8?q?=E2=9C=A8=20dashboard:=20workflow=20bundle?= =?UTF-8?q?=20export/import=20and=20hub=20marketplace=20proxy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /api/workflows/{name}/{version}/bundle exports a verified bundle; POST /api/workflows/import verifies and recompiles locally before saving (422 on verification failure). /api/marketplace/workflows proxies the hub index with an ETag cache and serves the cached copy with stale=true on hub outage; .../{name}/{version}/import and .../publish move bundles between the library and the hub with the spoke's heartbeat credential; /api/marketplace/refresh forwards an on-demand refresh. Owner-gated and audited where the store or hub is written. Shared index types live in pkg/workflowmodel/bundle so hub and spoke agree on the wire shape. Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --- dashboard/openapi.json | 477 ++++++++++++++++ src/pkg/dashboard/api_workflows.go | 1 + .../dashboard/api_workflows_marketplace.go | 518 ++++++++++++++++++ .../api_workflows_marketplace_test.go | 441 +++++++++++++++ src/pkg/workflowmodel/bundle/bundle_test.go | 22 + src/pkg/workflowmodel/bundle/catalog.go | 73 +++ 6 files changed, 1532 insertions(+) create mode 100644 src/pkg/dashboard/api_workflows_marketplace.go create mode 100644 src/pkg/dashboard/api_workflows_marketplace_test.go create mode 100644 src/pkg/workflowmodel/bundle/catalog.go diff --git a/dashboard/openapi.json b/dashboard/openapi.json index d9d38ba87a..72944b0d0f 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -36416,6 +36416,314 @@ } } } + }, + "/api/workflows/{name}/{version}/bundle": { + "get": { + "tags": [ + "Workflows" + ], + "summary": "Export a stored workflow model as a signed bundle", + "description": "Compiles name@version and wraps model, compiled profile and the sha256 of the canonical model JSON in a WorkflowBundle (RFC #17620 phase 7). Any role. 422 when the model does not compile.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Bundle", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowBundle" + } + } + } + }, + "404": { + "description": "Not found" + }, + "422": { + "description": "Model does not compile" + }, + "503": { + "description": "Workflow store not enabled" + } + } + } + }, + "/api/workflows/import": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Import a workflow bundle", + "description": "Verifies the bundle (hash over the canonical model, local recompile compared structurally with the published profile) and stores the model. The version is pinned to the bundle's when that slot is free, else the next integer version. The store recompiles; the published profile is never persisted. Owner role; audited as workflow_import.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowBundle" + } + } + } + }, + "responses": { + "201": { + "description": "Imported and recompiled locally", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowImportResult" + } + } + } + }, + "422": { + "description": "Bundle verification failed (hash or published profile does not match local compile)" + }, + "400": { + "description": "Bad request" + }, + "403": { + "description": "Owner role required" + }, + "503": { + "description": "Workflow store or marketplace not enabled" + }, + "413": { + "description": "Bundle too large" + } + } + } + }, + "/api/marketplace/workflows": { + "get": { + "tags": [ + "Workflows" + ], + "summary": "Marketplace catalogue index (proxied from the hub)", + "description": "Returns the hub's catalogue index. The spoke caches it for workflow.marketplace.refresh_interval and revalidates with If-None-Match; ?refresh=1 forces revalidation. When the hub is unreachable the response is 502 with the last cached index and stale=true.", + "parameters": [ + { + "name": "refresh", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "description": "any value forces revalidation with the hub" + } + ], + "responses": { + "200": { + "description": "Index", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MarketplaceIndexResponse" + } + } + } + }, + "404": { + "description": "Marketplace disabled (workflow.marketplace.enabled=false)" + }, + "502": { + "description": "Hub unreachable; body carries the cached index (possibly empty) with stale=true", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MarketplaceIndexResponse" + } + } + } + }, + "503": { + "description": "No hub URL configured" + } + } + } + }, + "/api/marketplace/workflows/{name}/{version}/import": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Import a catalogue entry from the hub", + "description": "Fetches name@version's bundle from the hub, verifies it locally and stores it exactly as POST /api/workflows/import does. Owner role; audited as workflow_import.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "Imported and recompiled locally", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowImportResult" + } + } + } + }, + "422": { + "description": "Bundle verification failed (hash or published profile does not match local compile)" + }, + "403": { + "description": "Owner role required" + }, + "404": { + "description": "Not in the marketplace or marketplace disabled" + }, + "502": { + "description": "Hub unreachable or returned an invalid bundle" + }, + "503": { + "description": "Workflow store not enabled or no hub URL" + } + } + } + }, + "/api/marketplace/workflows/publish": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Publish a library entry to the hub marketplace", + "description": "Exports name@version as a bundle (tier local-published unless the model is ingested-unreviewed) and posts it to the hub with the spoke's heartbeat credential. Owner role; audited as workflow_publish.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "name", + "version" + ], + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "201": { + "description": "Published", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "published": { + "type": "string", + "description": "name@version" + }, + "hash": { + "type": "string" + }, + "tier": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "Bad request" + }, + "403": { + "description": "Owner role required, or the hub refused the spoke credential" + }, + "404": { + "description": "Entry not found" + }, + "409": { + "description": "name@version already published with a different hash" + }, + "422": { + "description": "Model does not compile or hub rejected the bundle" + }, + "502": { + "description": "Hub unreachable" + }, + "503": { + "description": "Workflow store not enabled or no hub URL" + } + } + } + }, + "/api/marketplace/refresh": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Ask the hub to refresh its ingested catalogue", + "description": "Proxies POST /api/marketplace/refresh on the hub (optionally ?repo=owner/name). Owner role on the spoke; the hub applies its own admin policy. Audited as workflow_marketplace_refresh.", + "parameters": [ + { + "name": "repo", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "202": { + "description": "Refresh accepted by the hub" + }, + "403": { + "description": "Owner role required or hub refused" + }, + "404": { + "description": "Marketplace disabled" + }, + "502": { + "description": "Hub unreachable or failed" + }, + "503": { + "description": "No hub URL" + } + } + } } }, "components": { @@ -37407,6 +37715,175 @@ "$ref": "#/components/schemas/WorkflowEntry" } } + }, + "WorkflowBundle": { + "type": "object", + "description": "Portable workflow model envelope: model + the publisher's compiled profile + sha256 of the canonical model JSON. Importers verify the hash and recompile locally; the published profile is only ever compared, never trusted.", + "required": [ + "apiVersion", + "model", + "profile", + "hash" + ], + "properties": { + "apiVersion": { + "type": "string", + "enum": [ + "workflow.hivecommons.dev/v1alpha1" + ] + }, + "kind": { + "type": "string", + "enum": [ + "WorkflowBundle" + ] + }, + "model": { + "$ref": "#/components/schemas/WorkflowModel" + }, + "profile": { + "type": "object", + "description": "compiled profile (see WorkflowProfile)" + }, + "hash": { + "type": "string", + "description": "sha256 hex of the canonical (sorted-key) model JSON" + }, + "createdAt": { + "type": "string", + "format": "date-time" + }, + "createdBy": { + "type": "string" + }, + "tier": { + "type": "string", + "enum": [ + "built-in", + "curated", + "ingested-unreviewed", + "local", + "local-published" + ] + } + } + }, + "WorkflowImportResult": { + "type": "object", + "properties": { + "entry": { + "$ref": "#/components/schemas/WorkflowEntry" + }, + "verified": { + "type": "boolean" + }, + "source": { + "type": "string", + "description": "name@version in the marketplace, for marketplace imports" + } + } + }, + "MarketplaceIndexEntry": { + "type": "object", + "description": "One immutable catalogue row, keyed name@version plus hash.", + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + }, + "hash": { + "type": "string" + }, + "tier": { + "type": "string", + "enum": [ + "built-in", + "curated", + "ingested-unreviewed", + "local", + "local-published" + ] + }, + "description": { + "type": "string" + }, + "source": { + "type": "string", + "description": "repository or author the model came from" + }, + "states": { + "type": "integer" + }, + "columns": { + "type": "integer" + }, + "publishedAt": { + "type": "string", + "format": "date-time" + }, + "publishedBy": { + "type": "string" + }, + "sourceSHA": { + "type": "string", + "description": "commit the model was ingested from" + }, + "newerIngestAvailable": { + "type": "boolean", + "description": "set on curated entries when a later ingested-unreviewed version exists" + }, + "adoption": { + "type": "object", + "description": "reserved; empty today" + }, + "ratings": { + "type": "object", + "description": "reserved; empty today" + }, + "benchmarks": { + "type": "object", + "description": "reserved; empty today" + } + } + }, + "MarketplaceIndex": { + "type": "object", + "properties": { + "generatedAt": { + "type": "string", + "format": "date-time" + }, + "entries": { + "type": "array", + "items": { + "$ref": "#/components/schemas/MarketplaceIndexEntry" + } + } + } + }, + "MarketplaceIndexResponse": { + "type": "object", + "properties": { + "index": { + "$ref": "#/components/schemas/MarketplaceIndex" + }, + "stale": { + "type": "boolean", + "description": "true when served from cache because the hub was unreachable" + }, + "etag": { + "type": "string" + }, + "fetchedAt": { + "type": "string", + "format": "date-time" + }, + "error": { + "type": "string" + } + } } } } diff --git a/src/pkg/dashboard/api_workflows.go b/src/pkg/dashboard/api_workflows.go index 76f88eac14..d937eca0b9 100644 --- a/src/pkg/dashboard/api_workflows.go +++ b/src/pkg/dashboard/api_workflows.go @@ -31,6 +31,7 @@ func (s *Server) registerWorkflowRoutes() { s.mux.HandleFunc("POST /api/workflows/{name}/{version}/simulate", s.handleWorkflowSimulate) s.registerWorkflowEditorRoutes() s.registerWorkflowIngestSourceRoutes() + s.registerWorkflowMarketplaceRoutes() } func (s *Server) workflowStore(w http.ResponseWriter) *wfstore.Store { diff --git a/src/pkg/dashboard/api_workflows_marketplace.go b/src/pkg/dashboard/api_workflows_marketplace.go new file mode 100644 index 0000000000..98d2390836 --- /dev/null +++ b/src/pkg/dashboard/api_workflows_marketplace.go @@ -0,0 +1,518 @@ +package dashboard + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" + + spoke "github.com/hivecommons/hive/pkg/hub/spoke" + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" + wfstore "github.com/hivecommons/hive/pkg/workflowmodel/store" +) + +// RFC #17620 phase 7: bundle export/import on the spoke plus a proxy to the +// hub marketplace catalogue. Routes live in their own file so concurrent +// work on api_workflows.go only needs the one-line registration hook. + +const ( + // marketplaceHubTimeout bounds one spoke→hub call. + marketplaceHubTimeout = 20 * time.Second + // marketplaceMaxBody caps bundle and index payloads read from clients + // or the hub. + marketplaceMaxBody = 4 << 20 + // marketplaceHiveIDHeader carries the spoke's hive_id alongside the + // heartbeat bearer so the hub can bind the credential to a hive. + marketplaceHiveIDHeader = "X-Hive-ID" + // marketplaceStatusStale is the status returned when the hub is + // unreachable; the body may still carry a cached index. + marketplaceStatusStale = http.StatusBadGateway +) + +// marketplaceHTTP is the client used for hub calls; tests swap it. +var marketplaceHTTP = &http.Client{Timeout: marketplaceHubTimeout} + +// marketplaceIndexCache is the spoke's ETag-validated copy of the hub index. +type marketplaceIndexCache struct { + mu sync.Mutex + etag string + body []byte + fetchedAt time.Time +} + +var marketplaceCache marketplaceIndexCache + +func (s *Server) registerWorkflowMarketplaceRoutes() { + s.mux.HandleFunc("GET /api/workflows/{name}/{version}/bundle", s.handleWorkflowBundleGet) + s.mux.HandleFunc("POST /api/workflows/import", s.handleWorkflowImport) + s.mux.HandleFunc("GET /api/marketplace/workflows", s.handleMarketplaceIndex) + s.mux.HandleFunc("POST /api/marketplace/workflows/publish", s.handleMarketplacePublish) + s.mux.HandleFunc("POST /api/marketplace/workflows/{name}/{version}/import", s.handleMarketplaceImport) + s.mux.HandleFunc("POST /api/marketplace/refresh", s.handleMarketplaceRefresh) +} + +// marketplaceEnabled reports workflow.marketplace.enabled (default true). +func (s *Server) marketplaceEnabled() bool { + if s.deps == nil || s.deps.Config == nil { + return true + } + return s.deps.Config.Workflow.Marketplace.MarketplaceEnabled() +} + +// marketplaceBase is the hub base URL for /api/marketplace/*, or "" when the +// marketplace is disabled or no hub is configured. +func (s *Server) marketplaceBase() string { + if !s.marketplaceEnabled() || s.deps == nil || s.deps.Config == nil { + return "" + } + base := s.deps.Config.MarketplaceURL() + if u, err := url.Parse(base); err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return "" + } + return base +} + +func (s *Server) requireMarketplace(w http.ResponseWriter) (string, bool) { + if !s.marketplaceEnabled() { + jsonError(w, "workflow marketplace disabled (workflow.marketplace.enabled)", http.StatusNotFound) + return "", false + } + base := s.marketplaceBase() + if base == "" { + jsonError(w, "workflow marketplace has no hub URL (workflow.marketplace.url or hub.url)", http.StatusServiceUnavailable) + return "", false + } + return base, true +} + +// handleWorkflowBundleGet exports one stored model as a verified bundle. +// Any role: bundles contain nothing the library view does not already show. +func (s *Server) handleWorkflowBundleGet(w http.ResponseWriter, r *http.Request) { + st := s.workflowStore(w) + if st == nil { + return + } + b, err := s.exportBundle(st, r.PathValue("name"), r.PathValue("version"), requestUser(r)) + if err != nil { + if errors.Is(err, workflowmodel.ErrCompile) || errors.Is(err, workflowmodel.ErrInvalid) { + jsonError(w, err.Error(), http.StatusUnprocessableEntity) + return + } + workflowStoreError(w, err) + return + } + data, err := bundle.Encode(b) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(data) +} + +func (s *Server) exportBundle(st *wfstore.Store, name, version, by string) (*bundle.Bundle, error) { + m, _, err := st.Get(name, version) + if err != nil { + return nil, err + } + return bundle.Export(m, by, time.Now().UTC()) +} + +// handleWorkflowImport verifies a bundle posted by the client and stores its +// model, recompiling locally. 422 when the bundle does not verify. Owner. +func (s *Server) handleWorkflowImport(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + st := s.workflowStore(w) + if st == nil { + return + } + raw, err := io.ReadAll(io.LimitReader(r.Body, marketplaceMaxBody+1)) + if err != nil || len(raw) > marketplaceMaxBody { + jsonError(w, "bundle too large or unreadable", http.StatusRequestEntityTooLarge) + return + } + b, err := bundle.Decode(raw) + if err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + e, err := s.importBundle(st, b, requestUser(r)) + if err != nil { + var ve *importVerifyError + if errors.As(err, &ve) { + jsonError(w, ve.Error(), http.StatusUnprocessableEntity) + return + } + workflowStoreError(w, err) + return + } + s.auditFromRequest(r, "workflow_import", fmt.Sprintf("%s hash=%s tier=%s", e.Key(), shortHash(b.Hash), b.Tier), "") + jsonStatusResponse(w, http.StatusCreated, map[string]interface{}{"entry": e, "verified": true}) +} + +type importVerifyError struct{ err error } + +func (e *importVerifyError) Error() string { return "bundle verification failed: " + e.err.Error() } +func (e *importVerifyError) Unwrap() error { return e.err } + +// importBundle verifies b, pins the model to the bundle's version when that +// slot is free (otherwise the next integer version) and saves it. The +// store recompiles on Save, so the published profile is never persisted. +func (s *Server) importBundle(st *wfstore.Store, b *bundle.Bundle, by string) (wfstore.Entry, error) { + if err := bundle.Verify(b); err != nil { + return wfstore.Entry{}, &importVerifyError{err} + } + m := b.Model + name := m.Metadata.Name + version := m.Spec.Meta.Version + if wfstore.IsBuiltin(name) { + return wfstore.Entry{}, fmt.Errorf("%w: %s is a built-in model", wfstore.ErrReadOnly, name) + } + if version != "" { + if _, _, err := st.Get(name, version); err == nil { + version = "" + } else if !errors.Is(err, wfstore.ErrNotFound) { + return wfstore.Entry{}, err + } + } + if version == "" { + next, err := nextWorkflowVersion(st, name) + if err != nil { + return wfstore.Entry{}, err + } + version = next + } + m.Spec.Meta.Version = version + if m.Spec.Meta.Provenance == "" { + m.Spec.Meta.Provenance = workflowmodel.Provenance(b.Tier) + } + if m.Spec.Meta.Provenance == workflowmodel.ProvenanceBuiltIn || string(m.Spec.Meta.Provenance) == bundle.TierLocalPublished { + m.Spec.Meta.Provenance = workflowmodel.ProvenanceLocal + } + return st.Save(m, by, time.Now().UTC()) +} + +// hubRequest builds an authenticated spoke→hub request. +func (s *Server) hubRequest(ctx context.Context, method, rawURL string, body []byte) (*http.Request, error) { + var rd io.Reader + if body != nil { + rd = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, rawURL, rd) + if err != nil { + return nil, err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("Accept", "application/json") + if key := spoke.SpokeHeartbeatKey(); key != "" { + req.Header.Set("Authorization", "Bearer "+key) + } + if s.deps != nil && s.deps.Config != nil { + if id := strings.TrimSpace(s.deps.Config.HiveID); id != "" { + req.Header.Set(marketplaceHiveIDHeader, id) + } + } + return req, nil +} + +// handleMarketplaceIndex proxies the hub index with an ETag cache. When the +// hub is unreachable it answers 502 carrying the last cached index (if any) +// and stale:true so the UI can still render. +func (s *Server) handleMarketplaceIndex(w http.ResponseWriter, r *http.Request) { + base, ok := s.requireMarketplace(w) + if !ok { + return + } + ttl := s.deps.Config.Workflow.Marketplace.EffectiveRefreshInterval() + marketplaceCache.mu.Lock() + defer marketplaceCache.mu.Unlock() + c := &marketplaceCache + fresh := c.body != nil && time.Since(c.fetchedAt) < ttl && r.URL.Query().Get("refresh") == "" + if !fresh { + ctx, cancel := context.WithTimeout(r.Context(), marketplaceHubTimeout) + defer cancel() + req, err := s.hubRequest(ctx, http.MethodGet, base+"/api/marketplace/workflows", nil) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + if c.etag != "" { + req.Header.Set("If-None-Match", c.etag) + } + resp, err := marketplaceHTTP.Do(req) + if err != nil { + writeMarketplaceIndex(w, c, true, "hub unreachable: "+err.Error()) + return + } + defer closeHTTPBody(resp.Body) + switch resp.StatusCode { + case http.StatusNotModified: + c.fetchedAt = time.Now() + case http.StatusOK: + body, err := io.ReadAll(io.LimitReader(resp.Body, marketplaceMaxBody+1)) + if err != nil || len(body) > marketplaceMaxBody || !json.Valid(body) { + writeMarketplaceIndex(w, c, true, "hub index unreadable") + return + } + c.body = body + c.etag = resp.Header.Get("ETag") + c.fetchedAt = time.Now() + default: + writeMarketplaceIndex(w, c, true, fmt.Sprintf("hub returned HTTP %d", resp.StatusCode)) + return + } + } + writeMarketplaceIndex(w, c, false, "") +} + +// writeMarketplaceIndex wraps the cached hub index as +// {index, stale, etag, fetchedAt, error?}. +func writeMarketplaceIndex(w http.ResponseWriter, c *marketplaceIndexCache, stale bool, errMsg string) { + out := map[string]interface{}{"stale": stale} + if c.body != nil { + out["index"] = json.RawMessage(c.body) + out["etag"] = c.etag + out["fetchedAt"] = c.fetchedAt.UTC() + } else { + out["index"] = bundle.Index{Entries: []bundle.IndexEntry{}} + } + if errMsg != "" { + out["error"] = errMsg + } + if stale { + jsonStatusResponse(w, marketplaceStatusStale, out) + return + } + jsonResponse(w, out) +} + +// fetchHubBundle downloads name@version from the hub and decodes it. +func (s *Server) fetchHubBundle(ctx context.Context, base, name, version string) (*bundle.Bundle, int, error) { + u := base + "/api/marketplace/workflows/" + url.PathEscape(name) + "/" + url.PathEscape(version) + "/bundle" + req, err := s.hubRequest(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, http.StatusInternalServerError, err + } + resp, err := marketplaceHTTP.Do(req) + if err != nil { + return nil, http.StatusBadGateway, fmt.Errorf("hub unreachable: %w", err) + } + defer closeHTTPBody(resp.Body) + if resp.StatusCode == http.StatusNotFound { + return nil, http.StatusNotFound, fmt.Errorf("%s@%s not in marketplace", name, version) + } + if resp.StatusCode != http.StatusOK { + return nil, http.StatusBadGateway, fmt.Errorf("hub returned HTTP %d", resp.StatusCode) + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, marketplaceMaxBody+1)) + if err != nil || len(raw) > marketplaceMaxBody { + return nil, http.StatusBadGateway, errors.New("hub bundle unreadable or too large") + } + b, err := bundle.Decode(raw) + if err != nil { + return nil, http.StatusBadGateway, fmt.Errorf("hub bundle invalid: %w", err) + } + return b, http.StatusOK, nil +} + +// handleMarketplaceImport fetches a catalogue bundle and imports it. Owner. +func (s *Server) handleMarketplaceImport(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + base, ok := s.requireMarketplace(w) + if !ok { + return + } + st := s.workflowStore(w) + if st == nil { + return + } + name, version := r.PathValue("name"), r.PathValue("version") + ctx, cancel := context.WithTimeout(r.Context(), marketplaceHubTimeout) + defer cancel() + b, code, err := s.fetchHubBundle(ctx, base, name, version) + if err != nil { + jsonError(w, err.Error(), code) + return + } + if b.Model.Metadata.Name != name || b.Model.Spec.Meta.Version != version { + jsonError(w, fmt.Sprintf("hub returned %s for %s@%s", b.Key(), name, version), http.StatusBadGateway) + return + } + e, err := s.importBundle(st, b, requestUser(r)) + if err != nil { + var ve *importVerifyError + if errors.As(err, &ve) { + jsonError(w, ve.Error(), http.StatusUnprocessableEntity) + return + } + workflowStoreError(w, err) + return + } + s.auditFromRequest(r, "workflow_import", fmt.Sprintf("%s from marketplace %s@%s hash=%s tier=%s", e.Key(), name, version, shortHash(b.Hash), b.Tier), "") + jsonStatusResponse(w, http.StatusCreated, map[string]interface{}{"entry": e, "verified": true, "source": name + "@" + version}) +} + +// marketplacePublishRequest is the body of POST /api/marketplace/workflows/publish. +type marketplacePublishRequest struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// handleMarketplacePublish exports a library entry and posts it to the hub +// with the spoke's heartbeat credential. Owner; audited. +func (s *Server) handleMarketplacePublish(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + base, ok := s.requireMarketplace(w) + if !ok { + return + } + st := s.workflowStore(w) + if st == nil { + return + } + var req marketplacePublishRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + req.Name, req.Version = strings.TrimSpace(req.Name), strings.TrimSpace(req.Version) + if req.Name == "" || req.Version == "" { + jsonError(w, "name and version are required", http.StatusBadRequest) + return + } + b, err := s.exportBundle(st, req.Name, req.Version, requestUser(r)) + if err != nil { + if errors.Is(err, workflowmodel.ErrCompile) || errors.Is(err, workflowmodel.ErrInvalid) { + jsonError(w, err.Error(), http.StatusUnprocessableEntity) + return + } + workflowStoreError(w, err) + return + } + // A spoke publishes from its own library: the hub records such bundles + // as local-published whatever the model's provenance says. + if b.Tier != bundle.TierIngestedUnreviewed { + b.Tier = bundle.TierLocalPublished + } + payload, err := bundle.Encode(b) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + ctx, cancel := context.WithTimeout(r.Context(), marketplaceHubTimeout) + defer cancel() + hreq, err := s.hubRequest(ctx, http.MethodPost, base+"/api/marketplace/workflows", payload) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + resp, err := marketplaceHTTP.Do(hreq) + if err != nil { + jsonError(w, "hub unreachable: "+err.Error(), http.StatusBadGateway) + return + } + defer closeHTTPBody(resp.Body) + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + code := http.StatusBadGateway + switch resp.StatusCode { + case http.StatusConflict: + code = http.StatusConflict + case http.StatusUnauthorized, http.StatusForbidden: + code = http.StatusForbidden + case http.StatusUnprocessableEntity, http.StatusBadRequest: + code = http.StatusUnprocessableEntity + } + jsonError(w, fmt.Sprintf("hub refused publish (HTTP %d): %s", resp.StatusCode, strings.TrimSpace(hubErrorMessage(msg))), code) + return + } + marketplaceCache.mu.Lock() + marketplaceCache.fetchedAt = time.Time{} + marketplaceCache.mu.Unlock() + s.auditFromRequest(r, "workflow_publish", fmt.Sprintf("%s hash=%s tier=%s to %s", b.Key(), shortHash(b.Hash), b.Tier, base), "") + jsonStatusResponse(w, http.StatusCreated, map[string]interface{}{"published": b.Key(), "hash": b.Hash, "tier": b.Tier}) +} + +// handleMarketplaceRefresh asks the hub to re-ingest its corpus (or one +// ?repo=). Owner on the spoke; the hub applies its own admin policy and may +// refuse, which is surfaced as 403. +func (s *Server) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + base, ok := s.requireMarketplace(w) + if !ok { + return + } + u := base + "/api/marketplace/refresh" + if repo := strings.TrimSpace(r.URL.Query().Get("repo")); repo != "" { + u += "?repo=" + url.QueryEscape(repo) + } + ctx, cancel := context.WithTimeout(r.Context(), marketplaceHubTimeout) + defer cancel() + hreq, err := s.hubRequest(ctx, http.MethodPost, u, []byte("{}")) + if err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + resp, err := marketplaceHTTP.Do(hreq) + if err != nil { + jsonError(w, "hub unreachable: "+err.Error(), http.StatusBadGateway) + return + } + defer closeHTTPBody(resp.Body) + body, _ := io.ReadAll(io.LimitReader(resp.Body, marketplaceMaxBody)) + switch { + case resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden: + jsonError(w, "hub refused refresh: "+strings.TrimSpace(hubErrorMessage(body)), http.StatusForbidden) + return + case resp.StatusCode >= 300: + jsonError(w, fmt.Sprintf("hub refresh failed (HTTP %d): %s", resp.StatusCode, strings.TrimSpace(hubErrorMessage(body))), http.StatusBadGateway) + return + } + marketplaceCache.mu.Lock() + marketplaceCache.fetchedAt = time.Time{} + marketplaceCache.mu.Unlock() + s.auditFromRequest(r, "workflow_marketplace_refresh", "requested hub refresh "+r.URL.Query().Get("repo"), "") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(resp.StatusCode) + if json.Valid(body) { + _, _ = w.Write(body) + } else { + _, _ = w.Write([]byte(`{"accepted":true}`)) + } +} + +// hubErrorMessage extracts {"error": "..."} from a hub reply, else the raw text. +func hubErrorMessage(body []byte) string { + var e struct { + Error string `json:"error"` + } + if json.Unmarshal(body, &e) == nil && e.Error != "" { + return e.Error + } + return string(body) +} + +func shortHash(h string) string { + if len(h) > 12 { + return h[:12] + } + return h +} diff --git a/src/pkg/dashboard/api_workflows_marketplace_test.go b/src/pkg/dashboard/api_workflows_marketplace_test.go new file mode 100644 index 0000000000..4a5add4cb1 --- /dev/null +++ b/src/pkg/dashboard/api_workflows_marketplace_test.go @@ -0,0 +1,441 @@ +package dashboard + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" + wfstore "github.com/hivecommons/hive/pkg/workflowmodel/store" +) + +func resetMarketplaceCache(t *testing.T) { + t.Helper() + clear := func() { + marketplaceCache.mu.Lock() + marketplaceCache.body, marketplaceCache.etag, marketplaceCache.fetchedAt = nil, "", time.Time{} + marketplaceCache.mu.Unlock() + } + clear() + t.Cleanup(clear) +} + +func saveTechdocs(t *testing.T, s *Server) { + t.Helper() + m := techdocsModel(t) + if _, err := s.deps.WorkflowStore.Save(m, "andy", time.Now()); err != nil { + t.Fatal(err) + } +} + +// uncompilableTechdocs validates (so the store accepts it) but does not +// compile: its first gate names no releasing role. +func uncompilableTechdocs(t *testing.T) *workflowmodel.Model { + t.Helper() + m := techdocsModel(t) + m.Spec.Meta.Version = "9" + m.Spec.Gates[0].ReleasedBy = "" + return m +} + +func marketplaceServer(t *testing.T, hubURL string) *Server { + t.Helper() + resetMarketplaceCache(t) + s := workflowTestServer(t, true) + s.deps.Config.Workflow.Marketplace.URL = hubURL + s.deps.Config.HiveID = "spoke-1" + t.Setenv("HIVE_HUB_SECRET", "test-hub-secret-marketplace") + t.Setenv("HIVE_ID", "spoke-1") + return s +} + +// fakeHub is a minimal marketplace hub: one bundle, an ETag'd index, and a +// recorder of what the spoke sent. +type fakeHub struct { + t *testing.T + bundle []byte + index []byte + hits atomic.Int32 + notMod atomic.Int32 + published [][]byte + lastAuth string + lastHive string + refreshes atomic.Int32 + publishRC int + down bool +} + +func newFakeHub(t *testing.T) (*fakeHub, *httptest.Server) { + t.Helper() + b, err := bundle.Export(techdocsModel(t), "hub", time.Now()) + if err != nil { + t.Fatal(err) + } + b.Tier = bundle.TierCurated + raw, _ := bundle.Encode(b) + e := bundle.EntryFor(b, time.Now(), "hub", "") + idx, _ := json.Marshal(bundle.Index{GeneratedAt: time.Now(), Entries: []bundle.IndexEntry{e}}) + h := &fakeHub{t: t, bundle: raw, index: idx, publishRC: http.StatusCreated} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h.lastAuth = r.Header.Get("Authorization") + h.lastHive = r.Header.Get(marketplaceHiveIDHeader) + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/marketplace/workflows": + h.hits.Add(1) + if r.Header.Get("If-None-Match") == `"idx-1"` { + h.notMod.Add(1) + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", `"idx-1"`) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(h.index) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/bundle"): + if !strings.Contains(r.URL.Path, "/cncf-techdocs/1/") { + http.Error(w, `{"error":"not found"}`, http.StatusNotFound) + return + } + _, _ = w.Write(h.bundle) + case r.Method == http.MethodPost && r.URL.Path == "/api/marketplace/workflows": + body, _ := io.ReadAll(r.Body) + h.published = append(h.published, body) + w.WriteHeader(h.publishRC) + _, _ = w.Write([]byte(`{"error":"nope"}`)) + case r.Method == http.MethodPost && r.URL.Path == "/api/marketplace/refresh": + h.refreshes.Add(1) + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"accepted":true,"repo":"` + r.URL.Query().Get("repo") + `"}`)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return h, srv +} + +func TestWorkflowBundleGetAndImportRoundTrip(t *testing.T) { + s := marketplaceServer(t, "") + saveTechdocs(t, s) + + rec := wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/1/bundle", nil, false) + if rec.Code != http.StatusOK { + t.Fatalf("bundle get: %d %s", rec.Code, rec.Body.String()) + } + b, err := bundle.Decode(rec.Body.Bytes()) + if err != nil || bundle.Verify(b) != nil { + t.Fatalf("exported bundle does not verify: %v", err) + } + if rec := wfReq(s, http.MethodGet, "/api/workflows/missing/1/bundle", nil, false); rec.Code != http.StatusNotFound { + t.Fatalf("missing: %d", rec.Code) + } + + // Import into a fresh store: version 1 is free so it stays pinned. + s2 := marketplaceServer(t, "") + rec = wfReq(s2, http.MethodPost, "/api/workflows/import", rec.Body.String(), true) + if rec.Code != http.StatusCreated { + t.Fatalf("import: %d %s", rec.Code, rec.Body.String()) + } + var out struct { + Entry wfstore.Entry `json:"entry"` + Verified bool `json:"verified"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if !out.Verified || out.Entry.Key() != "cncf-techdocs@1" || !out.Entry.Compiled { + t.Fatalf("import result = %+v", out) + } + // Same bundle again: version 1 is taken, so it lands at 2. + raw, _ := bundle.Encode(b) + rec = wfReq(s2, http.MethodPost, "/api/workflows/import", string(raw), true) + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if rec.Code != http.StatusCreated || out.Entry.Version != "2" { + t.Fatalf("second import: %d %+v", rec.Code, out.Entry) + } + // Non-owner, bad JSON, tampered. + if rec := wfReq(s2, http.MethodPost, "/api/workflows/import", string(raw), false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner import: %d", rec.Code) + } + if rec := wfReq(s2, http.MethodPost, "/api/workflows/import", "{", true); rec.Code != http.StatusBadRequest { + t.Fatalf("bad json: %d", rec.Code) + } + b.Model.Spec.Meta.Description = "tampered" + raw, _ = bundle.Encode(b) + rec = wfReq(s2, http.MethodPost, "/api/workflows/import", string(raw), true) + if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "verification failed") { + t.Fatalf("tampered import: %d %s", rec.Code, rec.Body.String()) + } + // Built-in names are refused. + m := techdocsModel(t) + m.Metadata.Name = "hive-default" + bb, _ := bundle.Export(m, "x", time.Now()) + raw, _ = bundle.Encode(bb) + if rec := wfReq(s2, http.MethodPost, "/api/workflows/import", string(raw), true); rec.Code != http.StatusForbidden { + t.Fatalf("built-in import: %d %s", rec.Code, rec.Body.String()) + } + // Without a store. + noStore := workflowTestServer(t, false) + if rec := wfReq(noStore, http.MethodPost, "/api/workflows/import", string(raw), true); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no store: %d", rec.Code) + } + if rec := wfReq(noStore, http.MethodGet, "/api/workflows/x/1/bundle", nil, false); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no store get: %d", rec.Code) + } +} + +func TestWorkflowBundleGetRefusesUncompilable(t *testing.T) { + s := marketplaceServer(t, "") + m := uncompilableTechdocs(t) + if _, err := s.deps.WorkflowStore.Save(m, "andy", time.Now()); err != nil { + t.Fatal(err) + } + if rec := wfReq(s, http.MethodGet, "/api/workflows/cncf-techdocs/9/bundle", nil, false); rec.Code != http.StatusUnprocessableEntity { + t.Fatalf("uncompilable export: %d %s", rec.Code, rec.Body.String()) + } +} + +func TestMarketplaceIndexProxyCachesWithETag(t *testing.T) { + hub, srv := newFakeHub(t) + s := marketplaceServer(t, srv.URL) + + rec := wfReq(s, http.MethodGet, "/api/marketplace/workflows", nil, false) + if rec.Code != http.StatusOK { + t.Fatalf("index: %d %s", rec.Code, rec.Body.String()) + } + var out struct { + Stale bool `json:"stale"` + ETag string `json:"etag"` + Index bundle.Index `json:"index"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if out.Stale || out.ETag != `"idx-1"` || len(out.Index.Entries) != 1 || out.Index.Entries[0].Tier != bundle.TierCurated { + t.Fatalf("index out = %+v", out) + } + if hub.lastAuth == "" || !strings.HasPrefix(hub.lastAuth, "Bearer ") || hub.lastHive != "spoke-1" { + t.Fatalf("spoke credentials not sent: auth=%q hive=%q", hub.lastAuth, hub.lastHive) + } + // Within TTL: served from cache, no hub hit. + wfReq(s, http.MethodGet, "/api/marketplace/workflows", nil, false) + if hub.hits.Load() != 1 { + t.Fatalf("expected cached, hub hits=%d", hub.hits.Load()) + } + // ?refresh=1 revalidates with If-None-Match → 304. + rec = wfReq(s, http.MethodGet, "/api/marketplace/workflows?refresh=1", nil, false) + if rec.Code != http.StatusOK || hub.notMod.Load() != 1 { + t.Fatalf("revalidate: %d notMod=%d", rec.Code, hub.notMod.Load()) + } + // Hub down → 502 with cached index and stale:true. + srv.Close() + rec = wfReq(s, http.MethodGet, "/api/marketplace/workflows?refresh=1", nil, false) + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if rec.Code != http.StatusBadGateway || !out.Stale || len(out.Index.Entries) != 1 { + t.Fatalf("stale: %d %s", rec.Code, rec.Body.String()) + } +} + +func TestMarketplaceIndexProxyErrors(t *testing.T) { + // Hub returns 500 with an empty cache → 502, empty index, stale. + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer bad.Close() + s := marketplaceServer(t, bad.URL) + rec := wfReq(s, http.MethodGet, "/api/marketplace/workflows", nil, false) + if rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), `"stale":true`) || !strings.Contains(rec.Body.String(), `"entries":[]`) { + t.Fatalf("hub 500: %d %s", rec.Code, rec.Body.String()) + } + // Disabled → 404. + off := false + s.deps.Config.Workflow.Marketplace.Enabled = &off + if rec := wfReq(s, http.MethodGet, "/api/marketplace/workflows", nil, false); rec.Code != http.StatusNotFound { + t.Fatalf("disabled: %d", rec.Code) + } + s.deps.Config.Workflow.Marketplace.Enabled = nil + // No usable URL → 503. + s.deps.Config.Workflow.Marketplace.URL = "not a url" + s.deps.Config.Hub.URL = "" + if rec := wfReq(s, http.MethodGet, "/api/marketplace/workflows", nil, false); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no url: %d %s", rec.Code, rec.Body.String()) + } + // Garbage index body with empty cache → stale 502. + resetMarketplaceCache(t) + bad2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("not json")) })) + defer bad2.Close() + s.deps.Config.Workflow.Marketplace.URL = bad2.URL + if rec := wfReq(s, http.MethodGet, "/api/marketplace/workflows", nil, false); rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "unreadable") { + t.Fatalf("garbage: %d %s", rec.Code, rec.Body.String()) + } +} + +func TestMarketplaceImportFromHub(t *testing.T) { + hub, srv := newFakeHub(t) + s := marketplaceServer(t, srv.URL) + + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/cncf-techdocs/1/import", nil, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner: %d", rec.Code) + } + rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/cncf-techdocs/1/import", nil, true) + if rec.Code != http.StatusCreated { + t.Fatalf("import: %d %s", rec.Code, rec.Body.String()) + } + var out struct { + Entry wfstore.Entry `json:"entry"` + Source string `json:"source"` + } + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if out.Entry.Key() != "cncf-techdocs@1" || out.Source != "cncf-techdocs@1" { + t.Fatalf("out = %+v", out) + } + m, _, err := s.deps.WorkflowStore.Get("cncf-techdocs", "1") + if err != nil || m.Spec.Meta.Provenance != workflowmodel.ProvenanceCurated { + t.Fatalf("imported provenance = %v %v", m.Spec.Meta.Provenance, err) + } + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/nothere/1/import", nil, true); rec.Code != http.StatusNotFound { + t.Fatalf("missing on hub: %d %s", rec.Code, rec.Body.String()) + } + // Re-import lands on the next free version. + rec = wfReq(s, http.MethodPost, "/api/marketplace/workflows/cncf-techdocs/1/import", nil, true) + _ = json.Unmarshal(rec.Body.Bytes(), &out) + if rec.Code != http.StatusCreated || out.Entry.Version != "2" { + t.Fatalf("re-import: %d %+v", rec.Code, out.Entry) + } + // Tampered bundle on hub → 422. + b, _ := bundle.Decode(hub.bundle) + b.Profile.Columns = nil + hub.bundle, _ = bundle.Encode(b) + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/cncf-techdocs/1/import", nil, true); rec.Code != http.StatusUnprocessableEntity { + t.Fatalf("tampered hub bundle: %d %s", rec.Code, rec.Body.String()) + } + // Invalid bundle JSON on hub → 502. + hub.bundle = []byte("{") + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/cncf-techdocs/1/import", nil, true); rec.Code != http.StatusBadGateway { + t.Fatalf("invalid hub bundle: %d", rec.Code) + } + // Hub down → 502. + srv.Close() + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/cncf-techdocs/1/import", nil, true); rec.Code != http.StatusBadGateway { + t.Fatalf("hub down: %d", rec.Code) + } +} + +func TestMarketplaceImportKeyMismatch(t *testing.T) { + // A hub that answers every bundle path with the same bundle. + b, _ := bundle.Export(techdocsModel(t), "hub", time.Now()) + raw, _ := bundle.Encode(b) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(raw) })) + defer srv.Close() + s := marketplaceServer(t, srv.URL) + rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/other/7/import", nil, true) + if rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "hub returned cncf-techdocs@1") { + t.Fatalf("mismatch: %d %s", rec.Code, rec.Body.String()) + } +} + +func TestMarketplacePublish(t *testing.T) { + hub, srv := newFakeHub(t) + s := marketplaceServer(t, srv.URL) + saveTechdocs(t, s) + + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": "cncf-techdocs", "version": "1"}, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner: %d", rec.Code) + } + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": ""}, true); rec.Code != http.StatusBadRequest { + t.Fatalf("missing fields: %d", rec.Code) + } + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", "{", true); rec.Code != http.StatusBadRequest { + t.Fatalf("bad json: %d", rec.Code) + } + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": "nope", "version": "1"}, true); rec.Code != http.StatusNotFound { + t.Fatalf("unknown entry: %d", rec.Code) + } + rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": "cncf-techdocs", "version": "1"}, true) + if rec.Code != http.StatusCreated || !strings.Contains(rec.Body.String(), `"published":"cncf-techdocs@1"`) { + t.Fatalf("publish: %d %s", rec.Code, rec.Body.String()) + } + if len(hub.published) != 1 { + t.Fatalf("hub received %d bundles", len(hub.published)) + } + sent, err := bundle.Decode(hub.published[0]) + if err != nil || bundle.Verify(sent) != nil || sent.Tier != bundle.TierLocalPublished { + t.Fatalf("sent bundle = %v tier=%q", err, sent.Tier) + } + if !strings.HasPrefix(hub.lastAuth, "Bearer ") || hub.lastHive != "spoke-1" { + t.Fatalf("credentials: %q %q", hub.lastAuth, hub.lastHive) + } + // Hub refusals map onto spoke statuses. + for hubRC, want := range map[int]int{ + http.StatusConflict: http.StatusConflict, + http.StatusUnauthorized: http.StatusForbidden, + http.StatusForbidden: http.StatusForbidden, + http.StatusUnprocessableEntity: http.StatusUnprocessableEntity, + http.StatusInternalServerError: http.StatusBadGateway, + } { + hub.publishRC = hubRC + rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": "cncf-techdocs", "version": "1"}, true) + if rec.Code != want || !strings.Contains(rec.Body.String(), "nope") { + t.Fatalf("hub %d → %d (want %d): %s", hubRC, rec.Code, want, rec.Body.String()) + } + } + // Uncompilable entry → 422 before any hub call. + if _, err := s.deps.WorkflowStore.Save(uncompilableTechdocs(t), "andy", time.Now()); err != nil { + t.Fatal(err) + } + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": "cncf-techdocs", "version": "9"}, true); rec.Code != http.StatusUnprocessableEntity { + t.Fatalf("uncompilable publish: %d", rec.Code) + } + srv.Close() + if rec := wfReq(s, http.MethodPost, "/api/marketplace/workflows/publish", map[string]string{"name": "cncf-techdocs", "version": "1"}, true); rec.Code != http.StatusBadGateway { + t.Fatalf("hub down: %d", rec.Code) + } +} + +func TestMarketplaceRefreshProxy(t *testing.T) { + hub, srv := newFakeHub(t) + s := marketplaceServer(t, srv.URL) + if rec := wfReq(s, http.MethodPost, "/api/marketplace/refresh", nil, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner: %d", rec.Code) + } + rec := wfReq(s, http.MethodPost, "/api/marketplace/refresh?repo=org/repo", nil, true) + if rec.Code != http.StatusAccepted || !strings.Contains(rec.Body.String(), `"repo":"org/repo"`) || hub.refreshes.Load() != 1 { + t.Fatalf("refresh: %d %s", rec.Code, rec.Body.String()) + } + deny := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.RawQuery, "boom") { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte("plain text failure")) + return + } + http.Error(w, `{"error":"admin only"}`, http.StatusForbidden) + })) + defer deny.Close() + s.deps.Config.Workflow.Marketplace.URL = deny.URL + rec = wfReq(s, http.MethodPost, "/api/marketplace/refresh", nil, true) + if rec.Code != http.StatusForbidden || !strings.Contains(rec.Body.String(), "admin only") { + t.Fatalf("denied: %d %s", rec.Code, rec.Body.String()) + } + rec = wfReq(s, http.MethodPost, "/api/marketplace/refresh?repo=boom", nil, true) + if rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "plain text failure") { + t.Fatalf("hub 500: %d %s", rec.Code, rec.Body.String()) + } + srv.Close() + s.deps.Config.Workflow.Marketplace.URL = srv.URL + if rec := wfReq(s, http.MethodPost, "/api/marketplace/refresh", nil, true); rec.Code != http.StatusBadGateway { + t.Fatalf("hub down: %d", rec.Code) + } +} + +func TestMarketplaceBaseFallsBackToHubURL(t *testing.T) { + s := marketplaceServer(t, "") + s.deps.Config.Hub.URL = "https://hub.example/" + if got := s.marketplaceBase(); got != "https://hub.example" { + t.Fatalf("base = %q", got) + } + bare := &Server{} + if !bare.marketplaceEnabled() || bare.marketplaceBase() != "" { + t.Fatal("server without deps should be enabled but have no base") + } +} diff --git a/src/pkg/workflowmodel/bundle/bundle_test.go b/src/pkg/workflowmodel/bundle/bundle_test.go index f25f98b787..9a4913ac59 100644 --- a/src/pkg/workflowmodel/bundle/bundle_test.go +++ b/src/pkg/workflowmodel/bundle/bundle_test.go @@ -232,3 +232,25 @@ func mustJSON(t *testing.T, v interface{}) string { } return string(b) } + +func TestEntryFor(t *testing.T) { + e := EntryFor(nil, fixedNow, "andy", "") + if e.Name != "" || string(e.Adoption) != "{}" || string(e.Ratings) != "{}" || string(e.Benchmarks) != "{}" { + t.Fatalf("nil entry = %+v", e) + } + b, err := Export(techdocs(t), "andy", fixedNow) + if err != nil { + t.Fatal(err) + } + e = EntryFor(b, fixedNow, "andy", "abc123") + if e.Key() != "cncf-techdocs@1" || e.Hash != b.Hash || e.Tier != TierLocal || e.Source != "cncf/techdocs" || e.SourceSHA != "abc123" { + t.Fatalf("entry = %+v", e) + } + if e.States == 0 || e.Columns == 0 || e.Description == "" { + t.Fatalf("summary fields empty: %+v", e) + } + b.Model.Spec.Scope.Repository = "org/repo" + if EntryFor(b, fixedNow, "", "").Source != "org/repo" { + t.Fatal("repository scope should win as source") + } +} diff --git a/src/pkg/workflowmodel/bundle/catalog.go b/src/pkg/workflowmodel/bundle/catalog.go new file mode 100644 index 0000000000..10fda85323 --- /dev/null +++ b/src/pkg/workflowmodel/bundle/catalog.go @@ -0,0 +1,73 @@ +package bundle + +import ( + "encoding/json" + "time" +) + +// IndexEntry is one row of the marketplace catalogue index served by the hub +// at GET /api/marketplace/workflows and proxied by spokes. Entries are +// immutable once published: name@version plus Hash identify a bundle. +type IndexEntry struct { + Name string `json:"name"` + Version string `json:"version"` + Hash string `json:"hash"` + Tier string `json:"tier"` + Description string `json:"description,omitempty"` + Source string `json:"source,omitempty"` + States int `json:"states"` + Columns int `json:"columns"` + PublishedAt time.Time `json:"publishedAt"` + PublishedBy string `json:"publishedBy,omitempty"` + // SourceSHA is the commit the model was ingested from, for ingested tiers. + SourceSHA string `json:"sourceSHA,omitempty"` + // NewerIngestAvailable is set on curated entries when a later + // ingested-unreviewed version of the same name exists. + NewerIngestAvailable bool `json:"newerIngestAvailable"` + // Reserved for later phases; always present (empty objects) so clients + // can bind to them now. + Adoption json.RawMessage `json:"adoption"` + Ratings json.RawMessage `json:"ratings"` + Benchmarks json.RawMessage `json:"benchmarks"` +} + +// Key is `name@version`. +func (e IndexEntry) Key() string { return e.Name + "@" + e.Version } + +// EmptyObject is the JSON value of the reserved fields. +var EmptyObject = json.RawMessage(`{}`) + +// Index is the catalogue index document. +type Index struct { + GeneratedAt time.Time `json:"generatedAt"` + Entries []IndexEntry `json:"entries"` +} + +// EntryFor summarises a verified bundle as a catalogue row. +func EntryFor(b *Bundle, publishedAt time.Time, publishedBy, sourceSHA string) IndexEntry { + e := IndexEntry{ + PublishedAt: publishedAt.UTC(), + PublishedBy: publishedBy, + SourceSHA: sourceSHA, + Adoption: EmptyObject, + Ratings: EmptyObject, + Benchmarks: EmptyObject, + } + if b == nil || b.Model == nil { + return e + } + e.Name = b.Model.Metadata.Name + e.Version = b.Model.Spec.Meta.Version + e.Hash = b.Hash + e.Tier = b.Tier + e.Description = b.Model.Spec.Meta.Description + e.Source = b.Model.Spec.Meta.Author + if b.Model.Spec.Scope.Repository != "" { + e.Source = b.Model.Spec.Scope.Repository + } + e.States = len(b.Model.Spec.States) + if b.Profile != nil { + e.Columns = len(b.Profile.Columns) + } + return e +} From d888aca650bea643e7ec78d707e13a6a76a91413 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 11 Nov 2026 09:57:33 -0700 Subject: [PATCH 4/9] =?UTF-8?q?=E2=9C=A8=20hub:=20workflow=20marketplace?= =?UTF-8?q?=20catalogue=20with=20immutable=20bundles=20and=20corpus=20refr?= =?UTF-8?q?esh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an immutable, file-backed catalogue under the hub data dir (marketplace/workflows//.bundle.json + index.json) with routes for the index (strong ETag / 304), per-bundle fetch, publish (spoke heartbeat bearer or hub admin; curated/built-in tiers admin-only), admin on-demand refresh and a status view. The refresh walks the embedded CNCF corpus plus HIVE_HUB_MARKETPLACE_EXTRA_REPOS, records source SHA and pipeline-file hash per repo, skips unchanged repos and publishes changed ones as ingested-YYYYMMDD.N (tier ingested-unreviewed). Curated versions are never overwritten; the index flags them newerIngestAvailable. A nightly run (HIVE_HUB_MARKETPLACE_REFRESH_SCHEDULE, default 03:00 UTC) starts with the background pollers and seeds an empty catalogue once a GitHub token is present. Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --- src/docs/api-reference/hub-server.md | 5 + src/pkg/hub/background_pollers.go | 2 + src/pkg/hub/hub_testing_helpers_test.go | 1 + src/pkg/hub/marketplace.go | 302 +++++++++ src/pkg/hub/marketplace_refresh.go | 463 ++++++++++++++ src/pkg/hub/marketplace_store.go | 408 ++++++++++++ src/pkg/hub/marketplace_test.go | 793 ++++++++++++++++++++++++ src/pkg/hub/server.go | 8 + 8 files changed, 1982 insertions(+) create mode 100644 src/pkg/hub/marketplace.go create mode 100644 src/pkg/hub/marketplace_refresh.go create mode 100644 src/pkg/hub/marketplace_store.go create mode 100644 src/pkg/hub/marketplace_test.go diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index 3112db2c7d..65dd86f3f6 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -71,3 +71,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1791` | | `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1796` | | `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1798` | +| `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:34` | +| `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:35` | +| `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:36` | +| `POST` | `/api/marketplace/refresh` | Hub admin | On-demand ingest of the CNCF corpus (+ `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`). Without `?repo=` starts a background run → `202 {accepted, repos}` (`409` when one is already running); `?repo=owner/name` runs that repository synchronously → `200 {accepted, repo, result}` | `pkg/hub/marketplace.go:37` | +| `GET` | `/api/marketplace/status` | Public | `{entries, byTier, etag, generatedAt, corpusRepos, trackedRepos, failingRepos, refreshing, schedule, lastRefresh, lastCheckedAt}` — last refresh summary and counts by tier | `pkg/hub/marketplace.go:38` | diff --git a/src/pkg/hub/background_pollers.go b/src/pkg/hub/background_pollers.go index 4bcd0f8043..32629120f9 100644 --- a/src/pkg/hub/background_pollers.go +++ b/src/pkg/hub/background_pollers.go @@ -51,6 +51,8 @@ func (s *HubServer) StartBackgroundPollers(ctx context.Context) <-chan struct{} // Hourly commit of spoke benchmark records to hivecommons/benchmarks // (hivecommons/hive#17694). start(s.StartBenchmarksPublisher) + // Workflow marketplace nightly ingest + first-boot seed (#17620 phase 7). + start(s.StartMarketplaceRefresh) s.githubActivityMu.Lock() s.githubActivityCtx = ctx feed := s.githubActivityFeed diff --git a/src/pkg/hub/hub_testing_helpers_test.go b/src/pkg/hub/hub_testing_helpers_test.go index 21dc2b21e6..3cbbc972ff 100644 --- a/src/pkg/hub/hub_testing_helpers_test.go +++ b/src/pkg/hub/hub_testing_helpers_test.go @@ -92,6 +92,7 @@ var hubTestPathVars = []hubTestPathVar{ {ptr: &journeyStatePath, prod: journeyStatePath, rel: filepath.Join("saas", "journey-state.json")}, {ptr: &alertAcksPath, prod: alertAcksPath, rel: filepath.Join("saas", "hub-alert-acks.json")}, {ptr: &revokedSessionsPath, prod: revokedSessionsPath, rel: filepath.Join("saas", "hub-revoked-sessions.json")}, + {ptr: &hubMarketplaceDir, prod: hubMarketplaceDir, rel: "marketplace"}, } // hubTestConfig is the resolved set of NewHubServer arguments plus the one diff --git a/src/pkg/hub/marketplace.go b/src/pkg/hub/marketplace.go new file mode 100644 index 0000000000..3eb24ea673 --- /dev/null +++ b/src/pkg/hub/marketplace.go @@ -0,0 +1,302 @@ +package hub + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" +) + +// RFC #17620 phase 7: hub marketplace routes. +// +// GET /api/marketplace/workflows index (ETag / 304) +// GET /api/marketplace/workflows/{name}/{version}/bundle one bundle +// POST /api/marketplace/workflows publish (spoke bearer or admin) +// POST /api/marketplace/refresh[?repo=] on-demand ingest (admin) +// GET /api/marketplace/status last refresh + counts by tier +// +// Tier policy: a publisher may assert `local`, `local-published` or +// `ingested-unreviewed`; `curated` and `built-in` require a hub admin +// session. Spokes authenticate with their heartbeat bearer bound to the +// X-Hive-ID header, exactly as /api/nps does with hive_id in the body. + +const ( + marketplaceMaxBundleBytes = 4 << 20 + marketplaceHiveIDHeader = "X-Hive-ID" +) + +func (s *HubServer) registerMarketplaceRoutes() { + s.mux.HandleFunc("GET /api/marketplace/workflows", s.handleMarketplaceIndex) + s.mux.HandleFunc("GET /api/marketplace/workflows/{name}/{version}/bundle", s.handleMarketplaceBundle) + s.mux.HandleFunc("POST /api/marketplace/workflows", s.handleMarketplacePublish) + s.mux.HandleFunc("POST /api/marketplace/refresh", s.requireAdmin(s.handleMarketplaceRefresh)) + s.mux.HandleFunc("GET /api/marketplace/status", s.handleMarketplaceStatus) +} + +func marketplaceJSONError(w http.ResponseWriter, code int, msg string) { + writeJSONError(w, code, msg) +} + +func (s *HubServer) marketplaceGate(w http.ResponseWriter) bool { + if marketplaceDisabled() { + marketplaceJSONError(w, http.StatusNotFound, "marketplace disabled") + return false + } + return true +} + +// handleMarketplaceIndex serves the catalogue index with a strong ETag. +func (s *HubServer) handleMarketplaceIndex(w http.ResponseWriter, r *http.Request) { + if !s.marketplaceGate(w) { + return + } + _, raw, etag, err := s.marketplace().Index() + if err != nil { + marketplaceJSONError(w, http.StatusInternalServerError, "index unavailable: "+err.Error()) + return + } + w.Header().Set("ETag", etag) + w.Header().Set("Cache-Control", "no-cache") + if match := r.Header.Get("If-None-Match"); match != "" && marketplaceETagMatches(match, etag) { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(raw) +} + +func marketplaceETagMatches(header, etag string) bool { + for _, part := range strings.Split(header, ",") { + part = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(part), "W/")) + if part == "*" || part == etag { + return true + } + } + return false +} + +// handleMarketplaceBundle serves one immutable bundle. +func (s *HubServer) handleMarketplaceBundle(w http.ResponseWriter, r *http.Request) { + if !s.marketplaceGate(w) { + return + } + data, err := s.marketplace().Get(r.PathValue("name"), r.PathValue("version")) + switch { + case errors.Is(err, errMarketplaceNotFound): + marketplaceJSONError(w, http.StatusNotFound, "not found") + return + case errors.Is(err, errMarketplaceBadKey): + marketplaceJSONError(w, http.StatusBadRequest, err.Error()) + return + case err != nil: + marketplaceJSONError(w, http.StatusInternalServerError, err.Error()) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") + _, _ = w.Write(data) +} + +// marketplacePublisher identifies who may publish: a hub admin session or +// a registered spoke presenting its heartbeat bearer. +type marketplacePublisher struct { + name string + admin bool +} + +func (s *HubServer) marketplaceAuthenticate(r *http.Request) (marketplacePublisher, int, string) { + if auth := r.Header.Get("Authorization"); strings.HasPrefix(auth, "Bearer ") { + if s.hubSecret == "" { + return marketplacePublisher{}, http.StatusServiceUnavailable, "publishing requires a configured hub secret" + } + hiveID := sanitizeHeartbeatField(r.Header.Get(marketplaceHiveIDHeader)) + if hiveID == "" || hiveID != r.Header.Get(marketplaceHiveIDHeader) || !isValidName(hiveID) { + return marketplacePublisher{}, http.StatusBadRequest, "invalid " + marketplaceHiveIDHeader + } + if !s.verifyHeartbeatBearer(strings.TrimPrefix(auth, "Bearer "), hiveID) { + return marketplacePublisher{}, http.StatusUnauthorized, "unauthorized" + } + if !s.npsHiveRegistered(hiveID) { + return marketplacePublisher{}, http.StatusForbidden, "unknown hive - heartbeat first" + } + return marketplacePublisher{name: "hive:" + hiveID}, 0, "" + } + if !isCSRFSafe(r) { + return marketplacePublisher{}, http.StatusForbidden, "CSRF check failed" + } + user := s.getRealAuthUser(r) + if user == "" { + return marketplacePublisher{}, http.StatusUnauthorized, "unauthorized" + } + if !isHubAdmin(user) { + return marketplacePublisher{}, http.StatusForbidden, "admin access required to publish from a session; spokes publish with their heartbeat credential" + } + return marketplacePublisher{name: user, admin: true}, 0, "" +} + +// handleMarketplacePublish verifies and stores a bundle. +func (s *HubServer) handleMarketplacePublish(w http.ResponseWriter, r *http.Request) { + if !s.marketplaceGate(w) { + return + } + who, code, msg := s.marketplaceAuthenticate(r) + if code != 0 { + marketplaceJSONError(w, code, msg) + return + } + raw, err := io.ReadAll(io.LimitReader(r.Body, marketplaceMaxBundleBytes+1)) + if err != nil { + marketplaceJSONError(w, http.StatusBadRequest, "read error") + return + } + if len(raw) > marketplaceMaxBundleBytes { + marketplaceJSONError(w, http.StatusRequestEntityTooLarge, "bundle too large") + return + } + b, err := bundle.Decode(raw) + if err != nil { + marketplaceJSONError(w, http.StatusBadRequest, err.Error()) + return + } + switch b.Tier { + case "": + b.Tier = bundle.TierLocalPublished + case bundle.TierLocal: + b.Tier = bundle.TierLocalPublished + case bundle.TierLocalPublished, bundle.TierIngestedUnreviewed: + case bundle.TierCurated, bundle.TierBuiltIn: + if !who.admin { + marketplaceJSONError(w, http.StatusForbidden, fmt.Sprintf("tier %q requires a hub admin", b.Tier)) + return + } + default: + marketplaceJSONError(w, http.StatusBadRequest, "unknown tier "+b.Tier) + return + } + if err := bundle.Verify(b); err != nil { + marketplaceJSONError(w, http.StatusUnprocessableEntity, "bundle verification failed: "+err.Error()) + return + } + entry, created, err := s.marketplace().Publish(b, who.name, time.Now().UTC(), "") + switch { + case errors.Is(err, errMarketplaceConflict): + marketplaceJSONError(w, http.StatusConflict, err.Error()) + return + case errors.Is(err, errMarketplaceBadKey): + marketplaceJSONError(w, http.StatusBadRequest, err.Error()) + return + case err != nil: + marketplaceJSONError(w, http.StatusInternalServerError, err.Error()) + return + } + s.logger.Info("marketplace publish", "key", entry.Key(), "tier", entry.Tier, "by", who.name, "created", created, "hash", entry.Hash) + code = http.StatusOK + if created { + code = http.StatusCreated + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]interface{}{"published": entry.Key(), "created": created, "entry": entry}) +} + +// handleMarketplaceRefresh runs an on-demand ingest in the background and +// answers 202 immediately; ?repo= limits it to one repository (and runs +// synchronously, since a single repo is quick enough to await). +func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Request) { + if !s.marketplaceGate(w) { + return + } + repo := strings.ToLower(strings.TrimSpace(r.URL.Query().Get("repo"))) + user := s.getRealAuthUser(r) + if repo != "" { + if !strings.Contains(repo, "/") { + marketplaceJSONError(w, http.StatusBadRequest, "repo must be owner/name") + return + } + res, err := s.runMarketplaceRefresh(r.Context(), []string{repo}, "on-demand:"+user) + if errors.Is(err, errMarketplaceRefreshBusy) { + marketplaceJSONError(w, http.StatusConflict, err.Error()) + return + } + if err != nil { + marketplaceJSONError(w, http.StatusInternalServerError, err.Error()) + return + } + s.logger.Info("marketplace refresh (single repo)", "by", user, "repo", repo, "published", res.Published, "errors", res.Errors) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]interface{}{"accepted": true, "repo": repo, "result": res}) + return + } + rf := s.refresher() + rf.mu.Lock() + busy := rf.running + rf.mu.Unlock() + if busy { + marketplaceJSONError(w, http.StatusConflict, errMarketplaceRefreshBusy.Error()) + return + } + s.logger.Info("marketplace refresh requested", "by", user) + go func() { + if _, err := s.runMarketplaceRefresh(s.marketplaceBackgroundContext(), nil, "on-demand:"+user); err != nil { + s.logger.Warn("marketplace on-demand refresh skipped", "error", err) + } + }() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]interface{}{"accepted": true, "repos": len(marketplaceRepos())}) +} + +// handleMarketplaceStatus reports the last refresh and tier counts. +func (s *HubServer) handleMarketplaceStatus(w http.ResponseWriter, r *http.Request) { + if !s.marketplaceGate(w) { + return + } + idx, _, etag, err := s.marketplace().Index() + if err != nil { + marketplaceJSONError(w, http.StatusInternalServerError, err.Error()) + return + } + byTier := map[string]int{} + for _, e := range idx.Entries { + byTier[e.Tier]++ + } + rf := s.refresher() + rf.mu.Lock() + last := rf.last + running := rf.running + rf.mu.Unlock() + records := s.marketplace().refreshRecords() + var lastChecked time.Time + failing := 0 + for _, rec := range records { + if rec.CheckedAt.After(lastChecked) { + lastChecked = rec.CheckedAt + } + if rec.Error != "" { + failing++ + } + } + out := map[string]interface{}{ + "entries": len(idx.Entries), + "byTier": byTier, + "etag": etag, + "generatedAt": idx.GeneratedAt, + "corpusRepos": len(marketplaceRepos()), + "trackedRepos": len(records), + "failingRepos": failing, + "refreshing": running, + "schedule": marketplaceSchedule(), + "lastRefresh": last, + "lastCheckedAt": nil, + } + if !lastChecked.IsZero() { + out["lastCheckedAt"] = lastChecked + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(out) +} diff --git a/src/pkg/hub/marketplace_refresh.go b/src/pkg/hub/marketplace_refresh.go new file mode 100644 index 0000000000..d9951a30b4 --- /dev/null +++ b/src/pkg/hub/marketplace_refresh.go @@ -0,0 +1,463 @@ +package hub + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/evidence/complexity" + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest/corpus" +) + +// RFC #17620 phase 7: the marketplace's ingest refresh. +// +// Every run walks the CNCF survey corpus (pkg/workflowmodel/ingest/corpus) +// plus any operator-added repos, fetches each default branch, hashes the +// pipeline-file set and compares it with the last recorded run for that +// repository. Unchanged → no-op. Changed → Lifecycle → Compile → bundle → +// publish as `ingested-YYYYMMDD.N`, tier ingested-unreviewed. Curated +// versions are never touched; the index flags them newerIngestAvailable. + +const ( + // hubMarketplaceExtraReposEnv is a comma-separated list of extra + // owner/repo entries to ingest alongside the corpus. + hubMarketplaceExtraReposEnv = "HIVE_HUB_MARKETPLACE_EXTRA_REPOS" + // hubMarketplaceScheduleEnv is the daily refresh time as HH:MM (UTC). + // Empty disables the nightly run; on-demand refresh still works. + hubMarketplaceScheduleEnv = "HIVE_HUB_MARKETPLACE_REFRESH_SCHEDULE" + hubMarketplaceDefaultSchedule = "03:00" + // hubMarketplaceDisableEnv turns the whole catalogue off (routes 404). + hubMarketplaceDisableEnv = "HIVE_HUB_MARKETPLACE_DISABLED" + + marketplaceFetchTimeout = 2 * time.Minute + // marketplaceRateFloor is the X-RateLimit-Remaining below which a run + // waits for the reset (bounded by marketplaceRateMaxWait) before + // continuing. + marketplaceRateFloor = 50 + marketplaceRateMaxWait = 10 * time.Minute + marketplaceMaxTarball = 256 << 20 + marketplaceSeedDelay = 30 * time.Second +) + +// marketplaceRepoPause keeps the crawl polite: one repo at a time with a +// short gap, so a 300-repo corpus takes a few minutes and never bursts +// GitHub's secondary limits. A var so tests can shorten it. +var marketplaceRepoPause = 500 * time.Millisecond + +// marketplaceTreeFetcher fetches a repository's default branch and returns +// its pipeline files (ingest.ReadTree shape) plus the commit SHA. Tests +// supply a fake; production uses the GitHub tarball endpoint. +type marketplaceTreeFetcher func(ctx context.Context, repo string) (files map[string][]byte, sha string, err error) + +// marketplaceRefreshResult summarises one run. +type marketplaceRefreshResult struct { + StartedAt time.Time `json:"startedAt"` + FinishedAt time.Time `json:"finishedAt"` + Repos int `json:"repos"` + Unchanged int `json:"unchanged"` + Published []string `json:"published"` + Skipped int `json:"skipped"` + Errors []string `json:"errors"` + Trigger string `json:"trigger"` +} + +// marketplaceRefresher owns the run lock and the last-run summary. +type marketplaceRefresher struct { + mu sync.Mutex + running bool + last *marketplaceRefreshResult + fetch marketplaceTreeFetcher + // now is injectable for deterministic version names in tests. + now func() time.Time + // bgCtx bounds on-demand runs started from a request; set by + // StartMarketplaceRefresh so they stop with the other pollers. + bgCtx context.Context +} + +// marketplaceBackgroundContext is the pollers' context when running, else +// context.Background(). +func (s *HubServer) marketplaceBackgroundContext() context.Context { + rf := s.refresher() + rf.mu.Lock() + defer rf.mu.Unlock() + if rf.bgCtx != nil { + return rf.bgCtx + } + return context.Background() +} + +func (s *HubServer) marketplace() *marketplaceStore { + s.marketplaceOnce.Do(func() { + if s.marketplaceStore == nil { + s.marketplaceStore = newMarketplaceStore(hubMarketplaceDir) + } + if s.marketplaceRefresher == nil { + s.marketplaceRefresher = &marketplaceRefresher{fetch: s.marketplaceGitHubFetcher, now: time.Now} + } + }) + return s.marketplaceStore +} + +func (s *HubServer) refresher() *marketplaceRefresher { + s.marketplace() + return s.marketplaceRefresher +} + +// marketplaceRepos is the corpus plus HIVE_HUB_MARKETPLACE_EXTRA_REPOS, +// de-duplicated and sorted. +func marketplaceRepos() []string { + seen := map[string]bool{} + var out []string + add := func(r string) { + r = strings.ToLower(strings.TrimSpace(r)) + if r == "" || !strings.Contains(r, "/") || seen[r] { + return + } + seen[r] = true + out = append(out, r) + } + for _, r := range corpus.Repos() { + add(r) + } + for _, r := range strings.Split(os.Getenv(hubMarketplaceExtraReposEnv), ",") { + add(r) + } + sort.Strings(out) + return out +} + +// marketplaceDisabled reports HIVE_HUB_MARKETPLACE_DISABLED. +func marketplaceDisabled() bool { + v := strings.ToLower(strings.TrimSpace(os.Getenv(hubMarketplaceDisableEnv))) + return v == "1" || v == "true" || v == "yes" +} + +// Run ingests repos (all when nil) and returns the summary. A second +// caller while a run is in flight gets errMarketplaceRefreshBusy. +var errMarketplaceRefreshBusy = errors.New("marketplace: refresh already running") + +func (s *HubServer) runMarketplaceRefresh(ctx context.Context, repos []string, trigger string) (*marketplaceRefreshResult, error) { + st := s.marketplace() + rf := s.refresher() + rf.mu.Lock() + if rf.running { + rf.mu.Unlock() + return nil, errMarketplaceRefreshBusy + } + rf.running = true + rf.mu.Unlock() + defer func() { + rf.mu.Lock() + rf.running = false + rf.mu.Unlock() + }() + + if repos == nil { + repos = marketplaceRepos() + } + res := &marketplaceRefreshResult{StartedAt: rf.now().UTC(), Trigger: trigger, Published: []string{}, Errors: []string{}} + for i, repo := range repos { + if ctx.Err() != nil { + res.Errors = append(res.Errors, "cancelled: "+ctx.Err().Error()) + break + } + if i > 0 { + select { + case <-ctx.Done(): + case <-time.After(marketplaceRepoPause): + } + } + res.Repos++ + outcome, err := s.refreshOneRepo(ctx, st, rf, repo) + switch { + case err != nil: + res.Errors = append(res.Errors, repo+": "+err.Error()) + s.logger.Warn("marketplace refresh: repo failed", "repo", repo, "error", err) + case outcome == "": + res.Unchanged++ + case outcome == "skipped": + res.Skipped++ + default: + res.Published = append(res.Published, outcome) + } + } + res.FinishedAt = rf.now().UTC() + rf.mu.Lock() + rf.last = res + rf.mu.Unlock() + s.logger.Info("marketplace refresh finished", "trigger", trigger, "repos", res.Repos, "published", len(res.Published), "unchanged", res.Unchanged, "skipped", res.Skipped, "errors", len(res.Errors)) + return res, nil +} + +// refreshOneRepo returns the published name@version, "" when unchanged, or +// "skipped" when the repository has no pipelines or does not compile. +func (s *HubServer) refreshOneRepo(ctx context.Context, st *marketplaceStore, rf *marketplaceRefresher, repo string) (string, error) { + fctx, cancel := context.WithTimeout(ctx, marketplaceFetchTimeout) + defer cancel() + files, sha, err := rf.fetch(fctx, repo) + if err != nil { + _ = st.saveRefreshRecord(marketplaceRefreshRecord{Repo: repo, CheckedAt: rf.now().UTC(), Error: err.Error()}) + return "", err + } + hash := marketplaceFilesHash(files) + rec := marketplaceRefreshRecord{Repo: repo, SourceSHA: sha, FilesHash: hash, Files: len(files), CheckedAt: rf.now().UTC()} + if prev, ok := st.refreshRecord(repo); ok && prev.FilesHash == hash && prev.Error == "" { + rec.Published = prev.Published + _ = st.saveRefreshRecord(rec) + return "", nil + } + if len(files) == 0 { + _ = st.saveRefreshRecord(rec) + return "skipped", nil + } + m := ingest.Lifecycle(workflowmodel.Scope{Forge: "github.com", Repository: repo}, files) + if len(m.Spec.Pipelines) == 0 { + _ = st.saveRefreshRecord(rec) + return "skipped", nil + } + version, err := marketplaceNextIngestVersion(st, m.Metadata.Name, rf.now()) + if err != nil { + return "", err + } + m.Spec.Meta.Version = version + m.Spec.Meta.Provenance = workflowmodel.ProvenanceIngestedUnreviewed + if m.Metadata.Annotations == nil { + m.Metadata.Annotations = map[string]string{} + } + m.Metadata.Annotations["workflow.hivecommons.dev/source-sha"] = sha + b, err := bundle.Export(m, "marketplace-refresh", rf.now()) + if err != nil { + rec.Error = "does not compile: " + err.Error() + _ = st.saveRefreshRecord(rec) + return "skipped", nil + } + b.Tier = bundle.TierIngestedUnreviewed + entry, created, err := st.Publish(b, "marketplace-refresh", rf.now(), sha) + if err != nil { + return "", err + } + rec.Published = entry.Key() + if err := st.saveRefreshRecord(rec); err != nil { + return "", err + } + if !created { + return "", nil + } + return entry.Key(), nil +} + +// marketplaceNextIngestVersion yields `ingested-YYYYMMDD.N`, N being one +// past the highest suffix already published for that name and day. +func marketplaceNextIngestVersion(st *marketplaceStore, name string, now time.Time) (string, error) { + prefix := "ingested-" + now.UTC().Format("20060102") + "." + versions, err := st.Versions(name) + if err != nil { + return "", err + } + max := 0 + for _, e := range versions { + if strings.HasPrefix(e.Version, prefix) { + if n, err := strconv.Atoi(strings.TrimPrefix(e.Version, prefix)); err == nil && n > max { + max = n + } + } + } + return prefix + strconv.Itoa(max+1), nil +} + +// marketplaceGitHubFetcher downloads the default-branch tarball through the +// hub's GitHub identity and returns the pipeline files + commit SHA. +func (s *HubServer) marketplaceGitHubFetcher(ctx context.Context, repo string) (map[string][]byte, string, error) { + owner, name, ok := strings.Cut(repo, "/") + if !ok || owner == "" || name == "" { + return nil, "", fmt.Errorf("repo %q must be owner/name", repo) + } + client := hubGitHubHTTPClient() + base := strings.TrimRight(githubAPIBase, "/") + var meta struct { + DefaultBranch string `json:"default_branch"` + } + if err := marketplaceGitHubJSON(ctx, client, base+"/repos/"+owner+"/"+name, &meta); err != nil { + return nil, "", err + } + if meta.DefaultBranch == "" { + return nil, "", errors.New("no default branch") + } + var branch struct { + Commit struct { + SHA string `json:"sha"` + } `json:"commit"` + } + if err := marketplaceGitHubJSON(ctx, client, base+"/repos/"+owner+"/"+name+"/branches/"+meta.DefaultBranch, &branch); err != nil { + return nil, "", err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/repos/"+owner+"/"+name+"/tarball/"+meta.DefaultBranch, nil) + if err != nil { + return nil, "", err + } + authGitHubRequest(req) + resp, err := client.Do(req) + if err != nil { + return nil, "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) + return nil, "", fmt.Errorf("tarball: %s", resp.Status) + } + dir, err := os.MkdirTemp("", "hive-marketplace-") + if err != nil { + return nil, "", err + } + defer os.RemoveAll(dir) + top, err := complexity.ExtractTarGz(resp.Body, dir, marketplaceMaxTarball) + if err != nil { + return nil, "", err + } + files, err := ingest.ReadTree(top) + if err != nil { + return nil, "", err + } + return files, branch.Commit.SHA, nil +} + +// marketplaceGitHubJSON GETs a GitHub API resource, honouring rate-limit +// headers: when the remaining budget is under marketplaceRateFloor it +// sleeps until the reset (bounded) before returning. +func marketplaceGitHubJSON(ctx context.Context, client *http.Client, url string, out interface{}) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + req.Header.Set("Accept", "application/vnd.github+json") + authGitHubRequest(req) + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + marketplaceHonourRateLimit(ctx, resp.Header) + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("%s: %s", url, resp.Status) + } + return json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(out) +} + +// marketplaceSleep is swapped in tests. +var marketplaceSleep = func(ctx context.Context, d time.Duration) { + select { + case <-ctx.Done(): + case <-time.After(d): + } +} + +func marketplaceHonourRateLimit(ctx context.Context, h http.Header) { + rem, err := strconv.Atoi(strings.TrimSpace(h.Get("X-RateLimit-Remaining"))) + if err != nil || rem >= marketplaceRateFloor { + return + } + reset, err := strconv.ParseInt(strings.TrimSpace(h.Get("X-RateLimit-Reset")), 10, 64) + if err != nil { + return + } + wait := time.Until(time.Unix(reset, 0)) + if wait <= 0 { + return + } + if wait > marketplaceRateMaxWait { + wait = marketplaceRateMaxWait + } + marketplaceSleep(ctx, wait) +} + +// marketplaceNextRun returns the next occurrence of the HH:MM UTC schedule +// after now, or false when the schedule is empty/invalid. +func marketplaceNextRun(schedule string, now time.Time) (time.Time, bool) { + schedule = strings.TrimSpace(schedule) + if schedule == "" { + return time.Time{}, false + } + t, err := time.Parse("15:04", schedule) + if err != nil { + return time.Time{}, false + } + now = now.UTC() + next := time.Date(now.Year(), now.Month(), now.Day(), t.Hour(), t.Minute(), 0, 0, time.UTC) + if !next.After(now) { + next = next.Add(24 * time.Hour) + } + return next, true +} + +// marketplaceSchedule is the configured HH:MM, defaulting to 03:00 UTC when +// the env var is unset; an explicitly empty value disables the nightly run. +func marketplaceSchedule() string { + v, ok := os.LookupEnv(hubMarketplaceScheduleEnv) + if !ok { + return hubMarketplaceDefaultSchedule + } + return strings.TrimSpace(v) +} + +// StartMarketplaceRefresh runs the nightly ingest until ctx is cancelled, +// seeding the catalogue once at start when it is empty and the hub has a +// GitHub token. Started from StartBackgroundPollers. +func (s *HubServer) StartMarketplaceRefresh(ctx context.Context) { + if marketplaceDisabled() { + s.logger.Info("marketplace disabled", "env", hubMarketplaceDisableEnv) + return + } + rf := s.refresher() + rf.mu.Lock() + rf.bgCtx = ctx + rf.mu.Unlock() + schedule := marketplaceSchedule() + s.logger.Info("marketplace refresh started", "schedule_utc", schedule, "corpus_repos", len(marketplaceRepos())) + if s.marketplaceShouldSeed() { + s.logger.Warn("marketplace catalogue is empty: seeding from the CNCF corpus in the background", "delay", marketplaceSeedDelay.String()) + marketplaceSleep(ctx, marketplaceSeedDelay) + if ctx.Err() == nil { + if _, err := s.runMarketplaceRefresh(ctx, nil, "seed"); err != nil { + s.logger.Warn("marketplace seed did not run", "error", err) + } + } + } + for { + next, ok := marketplaceNextRun(schedule, time.Now()) + if !ok { + s.logger.Info("marketplace nightly refresh disabled (empty or invalid schedule)", "schedule", schedule) + <-ctx.Done() + return + } + marketplaceSleep(ctx, time.Until(next)) + if ctx.Err() != nil { + return + } + if _, err := s.runMarketplaceRefresh(ctx, nil, "nightly"); err != nil { + s.logger.Warn("marketplace nightly refresh skipped", "error", err) + } + } +} + +// marketplaceShouldSeed is true when the catalogue has no entries and the +// hub can reach GitHub with a token. +func (s *HubServer) marketplaceShouldSeed() bool { + if hubGitHubAuthTokenForRequest(context.Background()) == "" { + return false + } + idx, _, _, err := s.marketplace().Index() + return err == nil && len(idx.Entries) == 0 +} diff --git a/src/pkg/hub/marketplace_store.go b/src/pkg/hub/marketplace_store.go new file mode 100644 index 0000000000..5864ca561d --- /dev/null +++ b/src/pkg/hub/marketplace_store.go @@ -0,0 +1,408 @@ +package hub + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" +) + +// RFC #17620 phase 7: the hub-side workflow marketplace catalogue. +// +// Layout under hubMarketplaceDir: +// +// workflows//.bundle.json immutable published bundles +// index.json derived catalogue index +// refresh/__.json last ingest run per corpus repo +// +// Entries are keyed name@version and never overwritten: a republish with the +// same key and hash is a no-op, a different hash is a 409. Curated versions +// are never replaced by ingest; the index flags them newerIngestAvailable +// when a later ingested-unreviewed version of the same name exists. + +// hubMarketplaceDir is the catalogue root; tests redirect it. +var hubMarketplaceDir = "/data/marketplace" + +var ( + errMarketplaceConflict = errors.New("marketplace: name@version already published with a different hash") + errMarketplaceNotFound = errors.New("marketplace: bundle not found") + errMarketplaceBadKey = errors.New("marketplace: invalid name or version") + + marketplaceNameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,63}$`) + marketplaceVersionRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9.-]{0,31}$`) +) + +// marketplaceStore is the on-disk catalogue. +type marketplaceStore struct { + root string + mu sync.RWMutex + // idx is the cached index document; nil until first load. + idx *bundle.Index + idxRaw []byte + idxETag string +} + +func newMarketplaceStore(root string) *marketplaceStore { + return &marketplaceStore{root: root} +} + +func (st *marketplaceStore) workflowsDir() string { return filepath.Join(st.root, "workflows") } +func (st *marketplaceStore) refreshDir() string { return filepath.Join(st.root, "refresh") } +func (st *marketplaceStore) indexPath() string { return filepath.Join(st.root, "index.json") } + +func (st *marketplaceStore) bundlePath(name, version string) string { + return filepath.Join(st.workflowsDir(), name, version+".bundle.json") +} + +func marketplaceValidKey(name, version string) bool { + return marketplaceNameRe.MatchString(name) && marketplaceVersionRe.MatchString(version) +} + +// Publish stores a verified bundle. Same key + same hash is idempotent; +// same key + different hash is errMarketplaceConflict. +func (st *marketplaceStore) Publish(b *bundle.Bundle, by string, now time.Time, sourceSHA string) (bundle.IndexEntry, bool, error) { + if err := bundle.Verify(b); err != nil { + return bundle.IndexEntry{}, false, err + } + name, version := b.Model.Metadata.Name, b.Model.Spec.Meta.Version + if !marketplaceValidKey(name, version) { + return bundle.IndexEntry{}, false, fmt.Errorf("%w: %s@%s", errMarketplaceBadKey, name, version) + } + st.mu.Lock() + defer st.mu.Unlock() + if existing, err := st.readBundleLocked(name, version); err == nil { + if existing.Hash == b.Hash { + idx, err := st.loadIndexLocked() + if err != nil { + return bundle.IndexEntry{}, false, err + } + for _, e := range idx.Entries { + if e.Name == name && e.Version == version { + return e, false, nil + } + } + return bundle.EntryFor(existing, existing.CreatedAt, existing.CreatedBy, sourceSHA), false, nil + } + return bundle.IndexEntry{}, false, fmt.Errorf("%w: %s@%s", errMarketplaceConflict, name, version) + } else if !errors.Is(err, errMarketplaceNotFound) { + return bundle.IndexEntry{}, false, err + } + data, err := bundle.Encode(b) + if err != nil { + return bundle.IndexEntry{}, false, err + } + if err := marketplaceWriteAtomic(st.bundlePath(name, version), data); err != nil { + return bundle.IndexEntry{}, false, err + } + entry := bundle.EntryFor(b, now, by, sourceSHA) + idx, err := st.loadIndexLocked() + if err != nil { + return bundle.IndexEntry{}, false, err + } + // loadIndexLocked may have rebuilt from disk and already seen the + // bundle written above; replace rather than duplicate. + kept := idx.Entries[:0] + for _, e := range idx.Entries { + if e.Name != name || e.Version != version { + kept = append(kept, e) + } + } + idx.Entries = append(kept, entry) + if err := st.writeIndexLocked(idx, now); err != nil { + return bundle.IndexEntry{}, false, err + } + for _, e := range st.idx.Entries { + if e.Name == name && e.Version == version { + entry = e + } + } + return entry, true, nil +} + +// Get returns the stored bundle bytes for name@version. +func (st *marketplaceStore) Get(name, version string) ([]byte, error) { + if !marketplaceValidKey(name, version) { + return nil, fmt.Errorf("%w: %s@%s", errMarketplaceBadKey, name, version) + } + st.mu.RLock() + defer st.mu.RUnlock() + data, err := os.ReadFile(st.bundlePath(name, version)) + if errors.Is(err, os.ErrNotExist) { + return nil, errMarketplaceNotFound + } + return data, err +} + +func (st *marketplaceStore) readBundleLocked(name, version string) (*bundle.Bundle, error) { + data, err := os.ReadFile(st.bundlePath(name, version)) + if errors.Is(err, os.ErrNotExist) { + return nil, errMarketplaceNotFound + } + if err != nil { + return nil, err + } + return bundle.Decode(data) +} + +// Index returns the catalogue index, its canonical bytes and ETag. +func (st *marketplaceStore) Index() (*bundle.Index, []byte, string, error) { + st.mu.Lock() + defer st.mu.Unlock() + if _, err := st.loadIndexLocked(); err != nil { + return nil, nil, "", err + } + cp := *st.idx + cp.Entries = append([]bundle.IndexEntry(nil), st.idx.Entries...) + return &cp, st.idxRaw, st.idxETag, nil +} + +// Versions lists the published versions of name, oldest first by publish time. +func (st *marketplaceStore) Versions(name string) ([]bundle.IndexEntry, error) { + idx, _, _, err := st.Index() + if err != nil { + return nil, err + } + var out []bundle.IndexEntry + for _, e := range idx.Entries { + if e.Name == name { + out = append(out, e) + } + } + return out, nil +} + +// loadIndexLocked reads index.json (rebuilding from the bundle files when it +// is missing) into st.idx and returns it. +func (st *marketplaceStore) loadIndexLocked() (*bundle.Index, error) { + if st.idx != nil { + return st.idx, nil + } + data, err := os.ReadFile(st.indexPath()) + switch { + case err == nil: + var idx bundle.Index + if jerr := json.Unmarshal(data, &idx); jerr == nil { + st.setIndexLocked(&idx, data) + return st.idx, nil + } + // Corrupt index: fall through and rebuild. + case !errors.Is(err, os.ErrNotExist): + return nil, err + } + idx, err := st.rebuildLocked() + if err != nil { + return nil, err + } + if err := st.writeIndexLocked(idx, time.Now().UTC()); err != nil { + return nil, err + } + return st.idx, nil +} + +// rebuildLocked derives an index from the bundle files on disk. +func (st *marketplaceStore) rebuildLocked() (*bundle.Index, error) { + idx := &bundle.Index{Entries: []bundle.IndexEntry{}} + names, err := os.ReadDir(st.workflowsDir()) + if errors.Is(err, os.ErrNotExist) { + return idx, nil + } + if err != nil { + return nil, err + } + for _, n := range names { + if !n.IsDir() { + continue + } + files, err := os.ReadDir(filepath.Join(st.workflowsDir(), n.Name())) + if err != nil { + return nil, err + } + for _, f := range files { + if !strings.HasSuffix(f.Name(), ".bundle.json") { + continue + } + version := strings.TrimSuffix(f.Name(), ".bundle.json") + b, err := st.readBundleLocked(n.Name(), version) + if err != nil { + continue + } + idx.Entries = append(idx.Entries, bundle.EntryFor(b, b.CreatedAt, b.CreatedBy, "")) + } + } + return idx, nil +} + +// writeIndexLocked normalises, flags curated entries, persists and caches. +func (st *marketplaceStore) writeIndexLocked(idx *bundle.Index, now time.Time) error { + marketplaceNormaliseIndex(idx) + idx.GeneratedAt = now.UTC() + data, err := json.MarshalIndent(idx, "", " ") + if err != nil { + return err + } + if err := marketplaceWriteAtomic(st.indexPath(), data); err != nil { + return err + } + st.setIndexLocked(idx, data) + return nil +} + +func (st *marketplaceStore) setIndexLocked(idx *bundle.Index, raw []byte) { + if idx.Entries == nil { + idx.Entries = []bundle.IndexEntry{} + } + st.idx = idx + st.idxRaw = raw + sum := sha256.Sum256(raw) + st.idxETag = `"` + hex.EncodeToString(sum[:]) + `"` +} + +// marketplaceNormaliseIndex sorts entries (name, then publish time) and +// recomputes newerIngestAvailable: a curated entry is flagged when any +// ingested-unreviewed entry of the same name was published after it. +func marketplaceNormaliseIndex(idx *bundle.Index) { + sort.SliceStable(idx.Entries, func(i, j int) bool { + a, b := idx.Entries[i], idx.Entries[j] + if a.Name != b.Name { + return a.Name < b.Name + } + if !a.PublishedAt.Equal(b.PublishedAt) { + return a.PublishedAt.Before(b.PublishedAt) + } + return a.Version < b.Version + }) + latestIngest := map[string]time.Time{} + for _, e := range idx.Entries { + if e.Tier == bundle.TierIngestedUnreviewed && e.PublishedAt.After(latestIngest[e.Name]) { + latestIngest[e.Name] = e.PublishedAt + } + } + for i := range idx.Entries { + e := &idx.Entries[i] + if len(e.Adoption) == 0 { + e.Adoption = bundle.EmptyObject + } + if len(e.Ratings) == 0 { + e.Ratings = bundle.EmptyObject + } + if len(e.Benchmarks) == 0 { + e.Benchmarks = bundle.EmptyObject + } + e.NewerIngestAvailable = e.Tier == bundle.TierCurated && latestIngest[e.Name].After(e.PublishedAt) + } +} + +// marketplaceRefreshRecord is what one ingest run remembers about a repo. +type marketplaceRefreshRecord struct { + Repo string `json:"repo"` + SourceSHA string `json:"sourceSHA"` + FilesHash string `json:"filesHash"` + Files int `json:"files"` + CheckedAt time.Time `json:"checkedAt"` + // Published is the name@version this run produced, or "" when the run + // was a no-op or the repository had no pipelines. + Published string `json:"published,omitempty"` + Error string `json:"error,omitempty"` +} + +func marketplaceRepoSlug(repo string) string { + return strings.ReplaceAll(strings.ToLower(strings.TrimSpace(repo)), "/", "__") +} + +func (st *marketplaceStore) refreshRecord(repo string) (marketplaceRefreshRecord, bool) { + st.mu.RLock() + defer st.mu.RUnlock() + data, err := os.ReadFile(filepath.Join(st.refreshDir(), marketplaceRepoSlug(repo)+".json")) + if err != nil { + return marketplaceRefreshRecord{}, false + } + var rec marketplaceRefreshRecord + if json.Unmarshal(data, &rec) != nil { + return marketplaceRefreshRecord{}, false + } + return rec, true +} + +func (st *marketplaceStore) saveRefreshRecord(rec marketplaceRefreshRecord) error { + data, err := json.MarshalIndent(rec, "", " ") + if err != nil { + return err + } + st.mu.Lock() + defer st.mu.Unlock() + return marketplaceWriteAtomic(filepath.Join(st.refreshDir(), marketplaceRepoSlug(rec.Repo)+".json"), data) +} + +// refreshRecords lists every recorded run. +func (st *marketplaceStore) refreshRecords() []marketplaceRefreshRecord { + st.mu.RLock() + defer st.mu.RUnlock() + files, err := os.ReadDir(st.refreshDir()) + if err != nil { + return nil + } + var out []marketplaceRefreshRecord + for _, f := range files { + data, err := os.ReadFile(filepath.Join(st.refreshDir(), f.Name())) + if err != nil { + continue + } + var rec marketplaceRefreshRecord + if json.Unmarshal(data, &rec) == nil { + out = append(out, rec) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Repo < out[j].Repo }) + return out +} + +// marketplaceFilesHash is a stable digest of a pipeline-file set: sorted +// path + sha256(content) pairs, hashed again. +func marketplaceFilesHash(files map[string][]byte) string { + paths := make([]string, 0, len(files)) + for p := range files { + paths = append(paths, p) + } + sort.Strings(paths) + h := sha256.New() + for _, p := range paths { + sum := sha256.Sum256(files[p]) + fmt.Fprintf(h, "%s\x00%x\n", p, sum) + } + return hex.EncodeToString(h.Sum(nil)) +} + +func marketplaceWriteAtomic(path string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(path), ".tmp-*") + if err != nil { + return err + } + tmpName := tmp.Name() + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + _ = os.Remove(tmpName) + return err + } + if err := tmp.Close(); err != nil { + _ = os.Remove(tmpName) + return err + } + if err := os.Rename(tmpName, path); err != nil { + _ = os.Remove(tmpName) + return err + } + return nil +} diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go new file mode 100644 index 0000000000..17eee4532e --- /dev/null +++ b/src/pkg/hub/marketplace_test.go @@ -0,0 +1,793 @@ +package hub + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/bundle" +) + +const mktTestHive = "hive-mkt" + +// marketplaceTestHub returns a handler hub with an isolated marketplace dir, +// a registered spoke and a fixed clock; fetch is the fake tree fetcher. +func marketplaceTestHub(t *testing.T, fetch marketplaceTreeFetcher) *HubServer { + t.Helper() + cleanup := helperSetupTempDirs(t) + t.Cleanup(cleanup) + old := hubMarketplaceDir + hubMarketplaceDir = filepath.Join(t.TempDir(), "marketplace") + t.Cleanup(func() { hubMarketplaceDir = old }) + oldPause := marketplaceRepoPause + marketplaceRepoPause = 0 + t.Cleanup(func() { marketplaceRepoPause = oldPause }) + t.Setenv(hubMarketplaceDisableEnv, "") + t.Setenv(hubMarketplaceExtraReposEnv, "") + + s := newHandlerHub() + s.registry.Hives = append(s.registry.Hives, RegistryEntry{ID: mktTestHive, Name: "Hive", Online: true}) + clock := time.Date(2026, 3, 14, 12, 0, 0, 0, time.UTC) + s.marketplaceRefresher = &marketplaceRefresher{fetch: fetch, now: func() time.Time { return clock }} + mkUser(t, hubAdminUsername) + mkUser(t, "someuser") + return s +} + +func mktModel(t *testing.T) *workflowmodel.Model { + t.Helper() + raw, err := os.ReadFile("../workflowmodel/testdata/models/techdocs.json") + if err != nil { + t.Fatal(err) + } + var m workflowmodel.Model + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return &m +} + +func mktBundle(t *testing.T, tier string) *bundle.Bundle { + t.Helper() + b, err := bundle.Export(mktModel(t), "tester", time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatal(err) + } + if tier != "" { + b.Tier = tier + } + return b +} + +func mktBody(t *testing.T, b *bundle.Bundle) string { + t.Helper() + raw, err := bundle.Encode(b) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func mktPublishReq(body string) *http.Request { + req := httptest.NewRequest(http.MethodPost, "/api/marketplace/workflows", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + return req +} + +func mktSpokeReq(body string) *http.Request { + req := mktPublishReq(body) + req.Header.Set("Authorization", heartbeatBearer(testHubSecret, mktTestHive)) + req.Header.Set(marketplaceHiveIDHeader, mktTestHive) + return req +} + +func mktAdminReq(body string) *http.Request { + req := mktPublishReq(body) + req.AddCookie(testAuthCookie(hubAdminUsername)) + req.Header.Set("Origin", "http://localhost") + return req +} + +func mktDo(s *HubServer, req *http.Request) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + s.mux = http.NewServeMux() + s.registerMarketplaceRoutes() + s.mux.ServeHTTP(rec, req) + return rec +} + +// --- store --- + +func TestMarketplaceStoreImmutable(t *testing.T) { + st := newMarketplaceStore(filepath.Join(t.TempDir(), "m")) + b := mktBundle(t, bundle.TierLocalPublished) + now := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + e, created, err := st.Publish(b, "a", now, "") + if err != nil || !created || e.Key() != "cncf-techdocs@1" { + t.Fatalf("first publish: %v %v %s", err, created, e.Key()) + } + // Same hash → idempotent. + _, created, err = st.Publish(b, "b", now, "") + if err != nil || created { + t.Fatalf("idempotent republish: %v created=%v", err, created) + } + // Different content under the same key → refused. + m2 := mktModel(t) + m2.Spec.Meta.Description = "changed" + b2, err := bundle.Export(m2, "c", now) + if err != nil { + t.Fatal(err) + } + b2.Tier = bundle.TierLocalPublished + if _, _, err := st.Publish(b2, "c", now, ""); !errors.Is(err, errMarketplaceConflict) { + t.Fatalf("overwrite: want conflict, got %v", err) + } + // Bad keys. + m3 := mktModel(t) + m3.Metadata.Name = "../escape" + b3, err := bundle.Export(m3, "c", now) + if err != nil { + t.Fatal(err) + } + if _, _, err := st.Publish(b3, "c", now, ""); !errors.Is(err, errMarketplaceBadKey) { + t.Fatalf("bad key: %v", err) + } + if _, err := st.Get("x", "../y"); !errors.Is(err, errMarketplaceBadKey) { + t.Fatalf("get bad key: %v", err) + } + if _, err := st.Get("cncf-techdocs", "99"); !errors.Is(err, errMarketplaceNotFound) { + t.Fatalf("get missing: %v", err) + } + raw, err := st.Get("cncf-techdocs", "1") + if err != nil { + t.Fatal(err) + } + got, err := bundle.Decode(raw) + if err != nil || got.Hash != b.Hash { + t.Fatalf("stored bundle: %v", err) + } + // Index survives a deleted index.json (rebuild from bundles). + if err := os.Remove(st.indexPath()); err != nil { + t.Fatal(err) + } + st2 := newMarketplaceStore(st.root) + idx, _, etag, err := st2.Index() + if err != nil || len(idx.Entries) != 1 || !strings.HasPrefix(etag, `"`) { + t.Fatalf("rebuilt index: %v %+v %q", err, idx, etag) + } + vs, err := st2.Versions("cncf-techdocs") + if err != nil || len(vs) != 1 { + t.Fatalf("versions: %v %d", err, len(vs)) + } +} + +func TestMarketplaceNormaliseIndexFlagsCurated(t *testing.T) { + idx := &bundle.Index{Entries: []bundle.IndexEntry{ + {Name: "a", Version: "ingested-20260314.1", Tier: bundle.TierIngestedUnreviewed, PublishedAt: time.Date(2026, 3, 14, 0, 0, 0, 0, time.UTC)}, + {Name: "a", Version: "2", Tier: bundle.TierCurated, PublishedAt: time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)}, + {Name: "b", Version: "1", Tier: bundle.TierCurated, PublishedAt: time.Date(2026, 3, 20, 0, 0, 0, 0, time.UTC)}, + {Name: "b", Version: "ingested-20260310.1", Tier: bundle.TierIngestedUnreviewed, PublishedAt: time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC)}, + }} + marketplaceNormaliseIndex(idx) + byKey := map[string]bundle.IndexEntry{} + for _, e := range idx.Entries { + byKey[e.Key()] = e + if string(e.Adoption) == "" || string(e.Ratings) == "" || string(e.Benchmarks) == "" { + t.Errorf("%s: reserved objects not filled", e.Key()) + } + } + if !byKey["a@2"].NewerIngestAvailable { + t.Error("a@2 curated should be flagged: newer ingest exists") + } + if byKey["b@1"].NewerIngestAvailable { + t.Error("b@1 curated is newer than its ingest; must not be flagged") + } + if byKey["a@ingested-20260314.1"].NewerIngestAvailable { + t.Error("ingested entries are never flagged") + } + if idx.Entries[0].Name != "a" || idx.Entries[len(idx.Entries)-1].Name != "b" { + t.Error("index not sorted by name") + } +} + +func TestMarketplaceFilesHashStable(t *testing.T) { + a := marketplaceFilesHash(map[string][]byte{"x": []byte("1"), "y": []byte("2")}) + b := marketplaceFilesHash(map[string][]byte{"y": []byte("2"), "x": []byte("1")}) + c := marketplaceFilesHash(map[string][]byte{"x": []byte("1"), "y": []byte("3")}) + if a != b || a == c { + t.Fatalf("hash: %s %s %s", a, b, c) + } +} + +// --- routes --- + +func TestMarketplaceIndexETag304(t *testing.T) { + s := marketplaceTestHub(t, nil) + if _, _, err := s.marketplace().Publish(mktBundle(t, bundle.TierLocalPublished), "x", time.Now(), ""); err != nil { + t.Fatal(err) + } + rec := mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/workflows", nil)) + if rec.Code != 200 { + t.Fatalf("index: %d %s", rec.Code, rec.Body.String()) + } + etag := rec.Header().Get("ETag") + if etag == "" { + t.Fatal("missing ETag") + } + var idx bundle.Index + if err := json.Unmarshal(rec.Body.Bytes(), &idx); err != nil || len(idx.Entries) != 1 { + t.Fatalf("index body: %v %s", err, rec.Body.String()) + } + if idx.Entries[0].Tier != bundle.TierLocalPublished || string(idx.Entries[0].Adoption) != "{}" { + t.Fatalf("entry: %+v", idx.Entries[0]) + } + req := httptest.NewRequest(http.MethodGet, "/api/marketplace/workflows", nil) + req.Header.Set("If-None-Match", `W/"nope", `+etag) + if rec = mktDo(s, req); rec.Code != http.StatusNotModified { + t.Fatalf("If-None-Match: %d", rec.Code) + } + req.Header.Set("If-None-Match", `"other"`) + if rec = mktDo(s, req); rec.Code != 200 { + t.Fatalf("stale etag: %d", rec.Code) + } + + rec = mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/workflows/cncf-techdocs/1/bundle", nil)) + if rec.Code != 200 || !strings.Contains(rec.Header().Get("Cache-Control"), "immutable") { + t.Fatalf("bundle: %d %s", rec.Code, rec.Header().Get("Cache-Control")) + } + if rec = mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/workflows/cncf-techdocs/9/bundle", nil)); rec.Code != 404 { + t.Fatalf("missing bundle: %d", rec.Code) + } + if rec = mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/workflows/bad!/1/bundle", nil)); rec.Code != 400 { + t.Fatalf("bad key: %d", rec.Code) + } + + t.Setenv(hubMarketplaceDisableEnv, "1") + if rec = mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/workflows", nil)); rec.Code != 404 { + t.Fatalf("disabled: %d", rec.Code) + } +} + +func TestMarketplacePublishAuth(t *testing.T) { + s := marketplaceTestHub(t, nil) + body := mktBody(t, mktBundle(t, bundle.TierLocal)) + + // Anonymous. + anon := mktPublishReq(body) + anon.Header.Set("Origin", "http://localhost") + if rec := mktDo(s, anon); rec.Code != 401 { + t.Fatalf("anon: %d %s", rec.Code, rec.Body.String()) + } + // Non-admin session. + user := mktPublishReq(body) + user.AddCookie(testAuthCookie("someuser")) + user.Header.Set("Origin", "http://localhost") + if rec := mktDo(s, user); rec.Code != 403 { + t.Fatalf("non-admin: %d", rec.Code) + } + // Admin cookie without CSRF evidence. + noCSRF := mktPublishReq(body) + noCSRF.AddCookie(testAuthCookie(hubAdminUsername)) + noCSRF.Header.Set("Origin", "https://evil.example") + if rec := mktDo(s, noCSRF); rec.Code != 403 { + t.Fatalf("csrf: %d", rec.Code) + } + // Spoke with wrong bearer. + bad := mktSpokeReq(body) + bad.Header.Set("Authorization", heartbeatBearer("wrong", mktTestHive)) + if rec := mktDo(s, bad); rec.Code != 401 { + t.Fatalf("bad bearer: %d", rec.Code) + } + // Spoke missing hive id. + noID := mktSpokeReq(body) + noID.Header.Del(marketplaceHiveIDHeader) + if rec := mktDo(s, noID); rec.Code != 400 { + t.Fatalf("no hive id: %d", rec.Code) + } + // Unknown hive (valid bearer for an unregistered id). + unknown := mktSpokeReq(body) + unknown.Header.Set("Authorization", heartbeatBearer(testHubSecret, "ghost")) + unknown.Header.Set(marketplaceHiveIDHeader, "ghost") + if rec := mktDo(s, unknown); rec.Code != 403 { + t.Fatalf("unknown hive: %d", rec.Code) + } + // Spoke asserting curated → 403. + if rec := mktDo(s, mktSpokeReq(mktBody(t, mktBundle(t, bundle.TierCurated)))); rec.Code != 403 { + t.Fatalf("spoke curated: %d %s", rec.Code, rec.Body.String()) + } + // Spoke OK: `local` is normalised to local-published. + rec := mktDo(s, mktSpokeReq(body)) + if rec.Code != 201 { + t.Fatalf("spoke publish: %d %s", rec.Code, rec.Body.String()) + } + var out struct { + Published string `json:"published"` + Created bool `json:"created"` + Entry bundle.IndexEntry `json:"entry"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + if out.Published != "cncf-techdocs@1" || !out.Created || out.Entry.Tier != bundle.TierLocalPublished || out.Entry.PublishedBy != "hive:"+mktTestHive { + t.Fatalf("publish result: %+v", out) + } + // Idempotent republish → 200. + if rec = mktDo(s, mktSpokeReq(body)); rec.Code != 200 { + t.Fatalf("republish: %d", rec.Code) + } + // Conflicting content under the same key → 409. + conflict := mktBundle(t, bundle.TierLocalPublished) + conflict.Model.Spec.Meta.Description = "changed" + conflict.Hash = bundle.CanonicalHash(conflict.Model) + conflict.Profile = nil + cb, err := bundle.Export(conflict.Model, "t", time.Now()) + if err != nil { + t.Fatal(err) + } + cb.Tier = bundle.TierLocalPublished + if rec = mktDo(s, mktSpokeReq(mktBody(t, cb))); rec.Code != 409 { + t.Fatalf("conflict: %d %s", rec.Code, rec.Body.String()) + } + // Admin may publish curated (different version). + curModel := mktModel(t) + curModel.Spec.Meta.Version = "2" + cur, err := bundle.Export(curModel, "admin", time.Now()) + if err != nil { + t.Fatal(err) + } + cur.Tier = bundle.TierCurated + if rec = mktDo(s, mktAdminReq(mktBody(t, cur))); rec.Code != 201 { + t.Fatalf("admin curated: %d %s", rec.Code, rec.Body.String()) + } + // Tampered bundle → 422. + tampered := mktBundle(t, bundle.TierLocalPublished) + tampered.Model.Spec.Meta.Version = "3" + if rec = mktDo(s, mktSpokeReq(mktBody(t, tampered))); rec.Code != 422 { + t.Fatalf("tampered: %d %s", rec.Code, rec.Body.String()) + } + // Garbage / unknown tier / too large. + if rec = mktDo(s, mktSpokeReq("{nope")); rec.Code != 400 { + t.Fatalf("garbage: %d", rec.Code) + } + if rec = mktDo(s, mktSpokeReq(mktBody(t, mktBundle(t, "platinum")))); rec.Code != 400 { + t.Fatalf("unknown tier: %d", rec.Code) + } + if rec = mktDo(s, mktSpokeReq(strings.Repeat(" ", marketplaceMaxBundleBytes+2))); rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("too large: %d", rec.Code) + } + // No hub secret → 503 for spoke lane. + s.hubSecret = "" + if rec = mktDo(s, mktSpokeReq(body)); rec.Code != 503 { + t.Fatalf("no secret: %d", rec.Code) + } +} + +// --- refresh --- + +func mktFixtureTree(t *testing.T, dir string) map[string][]byte { + t.Helper() + root := filepath.Join("..", "workflowmodel", "testdata", "corpus", dir) + files := map[string][]byte{} + err := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + b, err := os.ReadFile(p) + if err != nil { + return err + } + rel, _ := filepath.Rel(root, p) + files[filepath.ToSlash(rel)] = b + return nil + }) + if err != nil { + t.Fatal(err) + } + return files +} + +func TestMarketplaceRefreshNoopVsNewVersion(t *testing.T) { + trees := map[string]map[string][]byte{ + "envoyproxy/envoy": mktFixtureTree(t, "envoyproxy__envoy"), + "example/empty": {}, + "example/broken": nil, + "konflux-ci/konflux": mktFixtureTree(t, "konflux-ci__konflux-ci"), + } + shas := map[string]string{"envoyproxy/envoy": "sha1", "example/empty": "e", "konflux-ci/konflux": "k"} + calls := 0 + fetch := func(ctx context.Context, repo string) (map[string][]byte, string, error) { + calls++ + if repo == "example/broken" { + return nil, "", errors.New("boom") + } + return trees[repo], shas[repo], nil + } + s := marketplaceTestHub(t, fetch) + repos := []string{"envoyproxy/envoy", "example/empty", "example/broken", "konflux-ci/konflux"} + + res, err := s.runMarketplaceRefresh(context.Background(), repos, "test") + if err != nil { + t.Fatal(err) + } + if len(res.Published) != 2 || res.Skipped != 1 || len(res.Errors) != 1 || res.Unchanged != 0 { + t.Fatalf("first run: %+v", res) + } + for _, k := range res.Published { + if !strings.HasSuffix(k, "@ingested-20260314.1") { + t.Errorf("version naming: %s", k) + } + } + idx, _, _, err := s.marketplace().Index() + if err != nil || len(idx.Entries) != 2 { + t.Fatalf("index after first run: %v %d", err, len(idx.Entries)) + } + for _, e := range idx.Entries { + if e.Tier != bundle.TierIngestedUnreviewed || e.SourceSHA == "" { + t.Errorf("entry: %+v", e) + } + } + + // Second run with identical trees → all unchanged, nothing published. + res, err = s.runMarketplaceRefresh(context.Background(), repos, "test") + if err != nil { + t.Fatal(err) + } + if len(res.Published) != 0 || res.Unchanged != 3 || len(res.Errors) != 1 { + t.Fatalf("second run: %+v", res) + } + + // Change a pipeline file → new immutable version .2; the old one stays. + trees["envoyproxy/envoy"][".github/workflows/envoy-release.yml"] = append(trees["envoyproxy/envoy"][".github/workflows/envoy-release.yml"], []byte("\n# touched\n")...) + shas["envoyproxy/envoy"] = "sha2" + res, err = s.runMarketplaceRefresh(context.Background(), []string{"envoyproxy/envoy"}, "test") + if err != nil { + t.Fatal(err) + } + if len(res.Published) != 1 || !strings.HasSuffix(res.Published[0], "@ingested-20260314.2") { + t.Fatalf("third run: %+v", res) + } + name := strings.Split(res.Published[0], "@")[0] + vs, err := s.marketplace().Versions(name) + if err != nil || len(vs) != 2 { + t.Fatalf("versions: %v %d", err, len(vs)) + } + + // A curated version of the same name is flagged, never overwritten. + curRaw, err := s.marketplace().Get(name, "ingested-20260314.1") + if err != nil { + t.Fatal(err) + } + cur, err := bundle.Decode(curRaw) + if err != nil { + t.Fatal(err) + } + cur.Model.Spec.Meta.Version = "1" + cur.Model.Spec.Meta.Provenance = workflowmodel.ProvenanceCurated + cur, err = bundle.Export(cur.Model, "admin", time.Date(2026, 3, 13, 0, 0, 0, 0, time.UTC)) + if err != nil { + t.Fatal(err) + } + cur.Tier = bundle.TierCurated + if _, _, err := s.marketplace().Publish(cur, "admin", time.Date(2026, 3, 13, 0, 0, 0, 0, time.UTC), ""); err != nil { + t.Fatal(err) + } + idx, _, _, err = s.marketplace().Index() + if err != nil { + t.Fatal(err) + } + var sawCurated bool + for _, e := range idx.Entries { + if e.Name == name && e.Tier == bundle.TierCurated { + sawCurated = true + if !e.NewerIngestAvailable { + t.Errorf("curated %s should be flagged newerIngestAvailable", e.Key()) + } + } + } + if !sawCurated { + t.Fatal("curated entry missing") + } + if calls != 4+4+1 { + t.Errorf("fetch calls: %d", calls) + } + + // Refresh records. + recs := s.marketplace().refreshRecords() + if len(recs) != 4 { + t.Fatalf("records: %d", len(recs)) + } + rec, ok := s.marketplace().refreshRecord("example/broken") + if !ok || rec.Error == "" { + t.Fatalf("broken record: %v %+v", ok, rec) + } +} + +func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { + fetch := func(ctx context.Context, repo string) (map[string][]byte, string, error) { + if repo != "konflux-ci/konflux-ci" { + return map[string][]byte{}, "none", nil // corpus repos: no pipelines → skipped + } + return mktFixtureTree(t, "konflux-ci__konflux-ci"), "k1", nil + } + s := marketplaceTestHub(t, fetch) + + // Non-admin / anon are refused by requireAdmin. + rec := mktDo(s, httptest.NewRequest(http.MethodPost, "/api/marketplace/refresh?repo=a/b", nil)) + if rec.Code != 401 && rec.Code != 403 { + t.Fatalf("anon refresh: %d", rec.Code) + } + admin := func(target string) *http.Request { + req := httptest.NewRequest(http.MethodPost, target, nil) + req.AddCookie(testAuthCookie(hubAdminUsername)) + req.Header.Set("Origin", "http://localhost") + return req + } + if rec = mktDo(s, admin("/api/marketplace/refresh?repo=nope")); rec.Code != 400 { + t.Fatalf("bad repo: %d", rec.Code) + } + rec = mktDo(s, admin("/api/marketplace/refresh?repo=Konflux-CI/konflux-ci")) + if rec.Code != 200 { + t.Fatalf("single repo: %d %s", rec.Code, rec.Body.String()) + } + var single struct { + Repo string `json:"repo"` + Result marketplaceRefreshResult `json:"result"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &single); err != nil { + t.Fatal(err) + } + if single.Repo != "konflux-ci/konflux-ci" || len(single.Result.Published) != 1 { + t.Fatalf("single result: %+v", single) + } + + // Full run while another is running → 409; otherwise 202. + rf := s.refresher() + rf.mu.Lock() + rf.running = true + rf.mu.Unlock() + if rec = mktDo(s, admin("/api/marketplace/refresh")); rec.Code != 409 { + t.Fatalf("busy: %d", rec.Code) + } + if rec = mktDo(s, admin("/api/marketplace/refresh?repo=x/y")); rec.Code != 409 { + t.Fatalf("busy single: %d", rec.Code) + } + rf.mu.Lock() + rf.running = false + rf.mu.Unlock() + t.Setenv(hubMarketplaceExtraReposEnv, "konflux-ci/konflux-ci") + if rec = mktDo(s, admin("/api/marketplace/refresh")); rec.Code != 202 { + t.Fatalf("full: %d %s", rec.Code, rec.Body.String()) + } + deadline := time.Now().Add(5 * time.Second) + for { + rf.mu.Lock() + last := rf.last + running := rf.running + rf.mu.Unlock() + if !running && last != nil && strings.HasPrefix(last.Trigger, "on-demand:") { + break + } + if time.Now().After(deadline) { + t.Fatal("background refresh did not finish") + } + time.Sleep(10 * time.Millisecond) + } + + rec = mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/status", nil)) + if rec.Code != 200 { + t.Fatalf("status: %d", rec.Code) + } + var status map[string]interface{} + if err := json.Unmarshal(rec.Body.Bytes(), &status); err != nil { + t.Fatal(err) + } + if status["entries"].(float64) != 1 || status["schedule"] != hubMarketplaceDefaultSchedule || status["lastRefresh"] == nil || status["lastCheckedAt"] == nil { + t.Fatalf("status body: %s", rec.Body.String()) + } + byTier := status["byTier"].(map[string]interface{}) + if byTier[bundle.TierIngestedUnreviewed].(float64) != 1 { + t.Fatalf("byTier: %v", byTier) + } +} + +func TestMarketplaceReposMergesExtra(t *testing.T) { + t.Setenv(hubMarketplaceExtraReposEnv, " Foo/Bar ,foo/bar, nope ,") + repos := marketplaceRepos() + n := 0 + for _, r := range repos { + if r == "foo/bar" { + n++ + } + if r == "nope" || r != strings.ToLower(strings.TrimSpace(r)) { + t.Errorf("bad repo passed through: %q", r) + } + } + if n != 1 || len(repos) < 200 { + t.Fatalf("repos: foo/bar×%d, total %d", n, len(repos)) + } +} + +func TestMarketplaceNextRun(t *testing.T) { + now := time.Date(2026, 3, 14, 12, 0, 0, 0, time.UTC) + if _, ok := marketplaceNextRun("", now); ok { + t.Error("empty schedule should disable") + } + if _, ok := marketplaceNextRun("25:99", now); ok { + t.Error("invalid schedule should disable") + } + next, ok := marketplaceNextRun("03:00", now) + if !ok || !next.Equal(time.Date(2026, 3, 15, 3, 0, 0, 0, time.UTC)) { + t.Errorf("03:00 after noon → tomorrow, got %v", next) + } + next, _ = marketplaceNextRun("13:30", now) + if !next.Equal(time.Date(2026, 3, 14, 13, 30, 0, 0, time.UTC)) { + t.Errorf("13:30 → today, got %v", next) + } + t.Setenv(hubMarketplaceScheduleEnv, "") + if marketplaceSchedule() != "" { + t.Error("explicit empty env must disable") + } + os.Unsetenv(hubMarketplaceScheduleEnv) + if marketplaceSchedule() != hubMarketplaceDefaultSchedule { + t.Error("unset env → default") + } +} + +func TestMarketplaceHonourRateLimit(t *testing.T) { + var slept time.Duration + old := marketplaceSleep + marketplaceSleep = func(ctx context.Context, d time.Duration) { slept += d } + t.Cleanup(func() { marketplaceSleep = old }) + + h := http.Header{} + h.Set("X-RateLimit-Remaining", "1000") + marketplaceHonourRateLimit(context.Background(), h) + if slept != 0 { + t.Fatal("plenty remaining must not sleep") + } + h.Set("X-RateLimit-Remaining", "3") + h.Set("X-RateLimit-Reset", "garbage") + marketplaceHonourRateLimit(context.Background(), h) + if slept != 0 { + t.Fatal("unparseable reset must not sleep") + } + h.Set("X-RateLimit-Reset", "1") + marketplaceHonourRateLimit(context.Background(), h) + if slept != 0 { + t.Fatal("past reset must not sleep") + } + h.Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(2*time.Hour).Unix(), 10)) + marketplaceHonourRateLimit(context.Background(), h) + if slept != marketplaceRateMaxWait { + t.Fatalf("far reset should cap at max wait, slept %v", slept) + } +} + +func TestMarketplaceStartRefreshDisabledAndSeedGate(t *testing.T) { + s := marketplaceTestHub(t, nil) + t.Setenv(hubMarketplaceDisableEnv, "1") + done := make(chan struct{}) + go func() { s.StartMarketplaceRefresh(context.Background()); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("disabled StartMarketplaceRefresh should return immediately") + } + t.Setenv(hubMarketplaceDisableEnv, "") + t.Setenv("GITHUB_TOKEN", "") + t.Setenv("HIVE_HUB_GITHUB_TOKEN", "") + if s.marketplaceShouldSeed() { + t.Error("no token → no seed") + } + + // Empty schedule: loop parks on ctx and exits on cancel. + t.Setenv(hubMarketplaceScheduleEnv, "") + ctx, cancel := context.WithCancel(context.Background()) + done = make(chan struct{}) + go func() { s.StartMarketplaceRefresh(ctx); close(done) }() + time.Sleep(20 * time.Millisecond) + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("StartMarketplaceRefresh did not stop on cancel") + } +} + +func mktTarball(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for name, body := range files { + if err := tw.WriteHeader(&tar.Header{Name: "repo-abc/" + name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestMarketplaceGitHubFetcher(t *testing.T) { + tree := mktFixtureTree(t, "konflux-ci__konflux-ci") + strTree := map[string]string{"README.md": "hi"} + for k, v := range tree { + strTree[k] = string(v) + } + tarball := mktTarball(t, strTree) + var failBranch bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-RateLimit-Remaining", "4000") + switch r.URL.Path { + case "/repos/acme/widgets": + _, _ = w.Write([]byte(`{"default_branch":"main"}`)) + case "/repos/acme/widgets/branches/main": + if failBranch { + http.Error(w, "nope", http.StatusBadGateway) + return + } + _, _ = w.Write([]byte(`{"commit":{"sha":"abc123"}}`)) + case "/repos/acme/widgets/tarball/main": + _, _ = w.Write(tarball) + case "/repos/acme/nobranch": + _, _ = w.Write([]byte(`{}`)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + oldBase := githubAPIBase + githubAPIBase = srv.URL + t.Cleanup(func() { githubAPIBase = oldBase }) + t.Setenv(hubGitHubTokenEnv, "pat-token") + + s := marketplaceTestHub(t, nil) + files, sha, err := s.marketplaceGitHubFetcher(context.Background(), "acme/widgets") + if err != nil { + t.Fatal(err) + } + if sha != "abc123" || len(files) != len(tree) { + t.Fatalf("fetch: sha=%s files=%d want %d", sha, len(files), len(tree)) + } + if _, ok := files["README.md"]; ok { + t.Error("ReadTree should only keep pipeline files") + } + if _, _, err := s.marketplaceGitHubFetcher(context.Background(), "acme/missing"); err == nil { + t.Error("404 repo should fail") + } + if _, _, err := s.marketplaceGitHubFetcher(context.Background(), "acme/nobranch"); err == nil { + t.Error("missing default branch should fail") + } + if _, _, err := s.marketplaceGitHubFetcher(context.Background(), "garbage"); err == nil { + t.Error("bad repo spec should fail") + } + failBranch = true + if _, _, err := s.marketplaceGitHubFetcher(context.Background(), "acme/widgets"); err == nil { + t.Error("branch lookup failure should fail") + } + + // With a token and an empty catalogue the seed gate opens. + if !s.marketplaceShouldSeed() { + t.Error("token + empty catalogue → should seed") + } +} diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 724c4c15e1..07b3e0232b 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1444,6 +1444,12 @@ type HubServer struct { // field to point at a fake GitHub API before that first call. dibsPublicOnce sync.Once dibsPublic *dibsPublicChecker + + // Workflow marketplace catalogue (#17620 phase 7, marketplace*.go). + // Created lazily by marketplace(); tests pre-set the store/refresher. + marketplaceOnce sync.Once + marketplaceStore *marketplaceStore + marketplaceRefresher *marketplaceRefresher } // HubBannerEntry stores an admin banner targeted at a specific hive. @@ -1803,6 +1809,8 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub // Fleet activity page (#16063): signed-in, read-only board over // GET /api/hub/activity. s.registerActivityPageRoutes() + // Workflow marketplace catalogue (#17620 phase 7). + s.registerMarketplaceRoutes() go s.saveLoop() go s.refreshActiveMyHivesSnapshotsLoop() From f5ac75db7e1643d537914330f9eeccd098748083 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 11 Nov 2026 10:05:44 -0700 Subject: [PATCH 5/9] =?UTF-8?q?=E2=9C=A8=20dashboard=20UI:=20Workflows=20?= =?UTF-8?q?=E2=86=92=20Marketplace=20view=20with=20import,=20publish=20and?= =?UTF-8?q?=20catalogue=20refresh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a Marketplace item to the Settings → Workflows nav that lists the hub catalogue through the spoke proxy (tier badges, search, tier filter, newer-ingest flag, reserved adoption/rating columns, stale banner when the hub is unreachable), imports entries into the local library (the spoke verifies and recompiles; nothing is bound) and lets owners publish a library model to the hub after confirmation. New files only, plus one-line hooks in config-workflows-library.js and the manifest. Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --- .../static-src/css/workflows-marketplace.css | 11 + .../static-src/js/config-workflows-library.js | 8 +- .../js/config-workflows-marketplace.js | 204 ++++++++++++++++ src/pkg/dashboard/static-src/manifest.txt | 2 + src/pkg/dashboard/static/index.html | 223 +++++++++++++++++- .../workflows_marketplace_ui_test.go | 127 ++++++++++ 6 files changed, 571 insertions(+), 4 deletions(-) create mode 100644 src/pkg/dashboard/static-src/css/workflows-marketplace.css create mode 100644 src/pkg/dashboard/static-src/js/config-workflows-marketplace.js create mode 100644 src/pkg/dashboard/workflows_marketplace_ui_test.go diff --git a/src/pkg/dashboard/static-src/css/workflows-marketplace.css b/src/pkg/dashboard/static-src/css/workflows-marketplace.css new file mode 100644 index 0000000000..0366bf5d66 --- /dev/null +++ b/src/pkg/dashboard/static-src/css/workflows-marketplace.css @@ -0,0 +1,11 @@ +/* Settings → Workflows → Marketplace (#17620 phase 7): catalogue table, tier badges, stale banner */ + .wf-prov-local-published { color: var(--accent); border-color: color-mix(in srgb, var(--accent) 40%, transparent); background: color-mix(in srgb, var(--accent) 10%, transparent); } + .wf-mkt-toolbar { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-3); margin: var(--sp-4) 0; } + .wf-mkt-toolbar .wf-mkt-search { flex: 1 1 220px; min-width: 180px; font: inherit; font-size: var(--fs-sm); color: var(--text); background: var(--bg-soft); border: var(--line-width) solid var(--border); border-radius: var(--r-sm); padding: var(--sp-2) var(--sp-3); } + .wf-mkt-toolbar .wf-mkt-tier { font: inherit; font-size: var(--fs-sm); color: var(--text); background: var(--bg-soft); border: var(--line-width) solid var(--border); border-radius: var(--r-sm); padding: var(--sp-2) var(--sp-3); } + .wf-mkt-count { font-size: var(--fs-sm); white-space: nowrap; } + .wf-mkt-sub { font-size: var(--fs-xs); margin-top: var(--sp-1); } + .wf-mkt-banner { font-size: var(--fs-sm); padding: var(--sp-3) var(--sp-4); border-radius: var(--r-sm); margin: var(--sp-3) 0; border: var(--line-width) solid var(--border); } + .wf-mkt-stale { color: var(--status-warn); border-color: color-mix(in srgb, var(--status-warn) 40%, transparent); background: color-mix(in srgb, var(--status-warn) 10%, transparent); } + .wf-mkt-flag { display: inline-block; margin-top: var(--sp-1); font-size: var(--fs-xs); font-weight: 700; color: var(--status-warn); white-space: nowrap; } + .wf-mkt-table td { vertical-align: top; } diff --git a/src/pkg/dashboard/static-src/js/config-workflows-library.js b/src/pkg/dashboard/static-src/js/config-workflows-library.js index 2c93e913cf..b4b9a4260a 100644 --- a/src/pkg/dashboard/static-src/js/config-workflows-library.js +++ b/src/pkg/dashboard/static-src/js/config-workflows-library.js @@ -10,7 +10,7 @@ // // All HTML builders are pure functions of their inputs so the node // harness (workflows_library_ui_test.go) can run them without a DOM. - const WF_LIB_VIEWS = ['library', 'detail', 'ingest', 'editor']; + const WF_LIB_VIEWS = ['library', 'detail', 'ingest', 'editor', 'marketplace']; const WF_PROVENANCE_LABELS = { 'built-in': 'built-in', 'curated': 'curated', @@ -274,7 +274,7 @@ } function wfLibNav(view) { - const items = [['library', 'Library'], ['ingest', 'Ingest'], ['editor', 'Editor']]; + const items = [['library', 'Library'], ['ingest', 'Ingest'], ['editor', 'Editor'], ['marketplace', 'Marketplace']]; return '
' + items.map(([id, label]) => `` ).join('') + '
'; @@ -299,6 +299,7 @@ : ``) : '', owner ? `` : '', ``, + typeof wfMktPublishButton === 'function' ? wfMktPublishButton(e, owner) : '', canDelete ? `` : (owner && !builtin && bound.length ? `bound` : ''), ].join(''); @@ -469,6 +470,7 @@ : ``) : '', owner ? `` : '', ``, + typeof wfMktPublishButton === 'function' ? wfMktPublishButton(e, owner) : '', owner && !builtin && !bound.length ? `` : '', ].join(''); return '
' + @@ -778,6 +780,8 @@ body = renderWorkflowIngest(_wfLib.ingest, owner); } else if (_wfLib.view === 'editor') { body = renderWorkflowEditor(owner); + } else if (_wfLib.view === 'marketplace') { + body = renderWorkflowMarketplaceView(owner); } else { body = renderWorkflowLibraryTable(_workflowState.workflows, _workflowState.bindings, owner); } diff --git a/src/pkg/dashboard/static-src/js/config-workflows-marketplace.js b/src/pkg/dashboard/static-src/js/config-workflows-marketplace.js new file mode 100644 index 0000000000..fbc7d75f3e --- /dev/null +++ b/src/pkg/dashboard/static-src/js/config-workflows-marketplace.js @@ -0,0 +1,204 @@ +// Settings → Workflows → Marketplace (hivecommons/hive#17620, RFC phase 7). + // + // Browses the hub's workflow-model catalogue through the spoke proxy + // (GET /api/marketplace/workflows, ETag-cached server-side), imports an + // entry into the local library (the spoke re-verifies the bundle and + // recompiles the model; the imported model is pinned name@version and + // no binding moves), and lets an owner publish a local model back to + // the hub. Catalogue tiers: built-in · curated · ingested · unreviewed · + // local-published. Ratings/adoption columns are reserved and render + // "—" until the hub fills them. + // + // Render functions are pure (node-testable in + // workflows_marketplace_ui_test.go); state lives in _wfMkt. + const WF_MKT_TIERS = ['built-in', 'curated', 'ingested-unreviewed', 'local-published']; + const WF_MKT_TIER_LABELS = { + 'built-in': 'built-in', + 'curated': 'curated', + 'ingested-unreviewed': 'ingested · unreviewed', + 'local-published': 'local-published', + }; + const WF_MKT_TIER_TIPS = { + 'built-in': 'Ships with hive; identical on every spoke.', + 'curated': 'Reviewed and vouched for by a hub maintainer. Never overwritten by ingest; a newer ingest only sets a flag.', + 'ingested-unreviewed': 'Derived nightly from a CNCF project\'s pipeline files; nobody has reviewed it. Compiles with warnings.', + 'local-published': 'Published by another hive from its local library. Verified by the hub, not reviewed.', + }; + const _wfMkt = { loaded: false, loading: false, index: null, stale: false, error: '', fetchedAt: '', search: '', tier: '', busy: '' }; + + function wfMktTierBadge(tier) { + const t = WF_MKT_TIER_LABELS[tier] ? tier : 'local-published'; + return `${esc(WF_MKT_TIER_LABELS[t])}`; + } + + // Reserved read-only cells: "—" until the hub populates the object. + function wfMktReservedCell(obj, key, suffix) { + if (!obj || typeof obj !== 'object' || obj[key] === undefined || obj[key] === null || obj[key] === '') return '—'; + return esc(String(obj[key])) + (suffix ? esc(suffix) : ''); + } + + // Case-insensitive match on name, description and source; tier filter + // exact. Pure. + function wfMktFilter(entries, search, tier) { + const q = String(search || '').trim().toLowerCase(); + return (Array.isArray(entries) ? entries : []).filter(e => { + if (!e) return false; + if (tier && e.tier !== tier) return false; + if (!q) return true; + return [e.name, e.version, e.description, e.source, e.publishedBy].some(v => String(v || '').toLowerCase().includes(q)); + }); + } + + // Is name@version already in the local library? Imported models keep + // the catalogue's version when free, so a match means "installed". + function wfMktInstalled(entry, workflows) { + return (Array.isArray(workflows) ? workflows : []).some(w => w && w.name === entry.name && w.version === entry.version); + } + + function renderWorkflowMarketplaceRows(entries, workflows, owner, busyKey) { + const list = Array.isArray(entries) ? entries.slice() : []; + list.sort((a, b) => { + const ta = WF_MKT_TIERS.indexOf(a.tier), tb = WF_MKT_TIERS.indexOf(b.tier); + if (ta !== tb) return (ta < 0 ? 99 : ta) - (tb < 0 ? 99 : tb); + return (a.name + '@' + a.version).localeCompare(b.name + '@' + b.version); + }); + if (!list.length) return 'No catalogue entries match.'; + return list.map(e => { + const key = e.name + '@' + e.version; + const installed = wfMktInstalled(e, workflows); + const busy = busyKey === key; + let action; + if (installed) action = ``; + else if (owner) action = ``; + else action = '—'; + return `` + + `${esc(key)}${e.description ? `
${esc(e.description)}
` : ''}` + + `${wfMktTierBadge(e.tier)}${e.newerIngestAvailable ? '
newer ingest available
' : ''}` + + `${esc(e.source || '')}${e.sourceSHA ? `
${esc(String(e.sourceSHA).slice(0, 12))}
` : ''}` + + `${esc(String(e.states || 0))} / ${esc(String(e.columns || 0))}` + + `${e.publishedAt ? esc(formatDashboardDateTime(e.publishedAt)) : '—'}${e.publishedBy ? `
${esc(e.publishedBy)}
` : ''}` + + `${wfMktReservedCell(e.adoption, 'hives')}` + + `${wfMktReservedCell(e.ratings, 'average')}` + + `${installed ? 'in library' : '—'}` + + `${action}`; + }).join(''); + } + + function renderWorkflowMarketplace(state, workflows, owner) { + const st = state || {}; + const entries = (st.index && Array.isArray(st.index.entries)) ? st.index.entries : []; + const shown = wfMktFilter(entries, st.search, st.tier); + const counts = {}; + entries.forEach(e => { counts[e.tier] = (counts[e.tier] || 0) + 1; }); + let banner = ''; + if (st.stale) banner = `
Showing the last catalogue this hive cached${st.fetchedAt ? ' (' + esc(formatDashboardDateTime(st.fetchedAt)) + ')' : ''}: the hub could not be reached${st.error ? ' — ' + esc(st.error) : ''}.
`; + else if (st.error) banner = ``; + const tierOpts = [''].concat(WF_MKT_TIERS.map(t => + ``)).join(''); + const toolbar = '
' + + `` + + `` + + `${shown.length} of ${entries.length}` + + `` + + (owner ? `` : '') + + '
'; + let body; + if (!st.loaded && st.loading) body = '

Loading the catalogue…

'; + else body = '' + + '' + renderWorkflowMarketplaceRows(shown, workflows, owner, st.busy) + '
ModelTierSourceS / CPublishedAdoptionRatingLocal
'; + return '
' + + '

Marketplace

' + + '

Workflow models published to the hub: built-in and curated lifecycles, nightly ingests of CNCF project pipelines (unreviewed), and models other hives published. Importing verifies the bundle\'s hash, recompiles the model on this hive and saves it pinned to its catalogue version; no repository is bound and nothing moves automatically. The hub to browse is ' + yamlHint('workflow.marketplace.url') + ' (defaults to this hive\'s hub); the spoke re-checks its cached copy every ' + yamlHint('workflow.marketplace.refresh_interval') + '. Turn the catalogue off with ' + yamlHint('workflow.marketplace.enabled') + '.

' + + banner + toolbar + body + '
'; + } + + // Publish button for library rows and the detail view; '' unless the + // caller is an owner and the model is publishable (not built-in). + function wfMktPublishButton(entry, owner) { + if (!owner || !entry || wfProvenanceOf(entry) === 'built-in' || entry.compiled === false) return ''; + return ``; + } + + function wfMktRerender() { if (typeof wfLibRerender === 'function') wfLibRerender(); } + function wfMktSetSearch(value) { _wfMkt.search = String(value || ''); wfMktRerender(); } + function wfMktSetTier(value) { _wfMkt.tier = String(value || ''); wfMktRerender(); } + + async function wfMktLoad(force) { + if (_wfMkt.loading) return; + _wfMkt.loading = true; _wfMkt.error = ''; + wfMktRerender(); + try { + const res = await fetch('/api/marketplace/workflows' + (force ? '?refresh=1' : ''), { headers: _inceptionAuthHeaders() }); + let data = null; + try { data = await res.json(); } catch (e) { data = null; } + if (!res.ok && !(data && data.index)) throw new Error((data && data.error) || ('HTTP ' + res.status)); + _wfMkt.index = data.index || { entries: [] }; + _wfMkt.stale = !!data.stale; + _wfMkt.fetchedAt = data.fetchedAt || ''; + _wfMkt.error = data.error || ''; + _wfMkt.loaded = true; + } catch (err) { + _wfMkt.error = 'Could not load the catalogue: ' + err.message; + _wfMkt.loaded = true; + } + _wfMkt.loading = false; + wfMktRerender(); + } + + function wfMktReload() { wfMktLoad(true); } + + async function wfMktImport(name, version) { + if (!dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')) { showToast('Owner role is required to import.', 'error'); return; } + const key = name + '@' + version; + _wfMkt.busy = key; wfMktRerender(); + try { + const res = await fetch('/api/marketplace/workflows/' + encodeURIComponent(name) + '/' + encodeURIComponent(version) + '/import', { method: 'POST', headers: _inceptionAuthHeaders() }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + const data = await res.json(); + const e = data.entry || {}; + showToast('Imported ' + key + ' as ' + e.name + '@' + e.version + ' (verified, compiled locally)'); + _wfMkt.busy = ''; + if (typeof _workflowState !== 'undefined') _workflowState.loaded = false; + if (typeof loadWorkflowLibrary === 'function') await loadWorkflowLibrary(); + openWorkflowLibraryEntry(e.name, e.version); + } catch (err) { + _wfMkt.busy = ''; + showToast('Import failed: ' + err.message, 'error'); + } + wfMktRerender(); + } + + async function wfMktPublish(name, version) { + if (!dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')) { showToast('Owner role is required to publish.', 'error'); return; } + const ok = await hiveConfirm({ title: 'Publish ' + name + '@' + version + ' to the hub?', message: 'A signed bundle (model + compiled profile + hash) is sent to the hub marketplace under this hive\'s identity as tier local-published. Catalogue entries are immutable: once published, this name@version cannot be changed, only superseded by a new version.', confirmLabel: 'Publish' }); + if (!ok) return; + try { + const res = await fetch('/api/marketplace/workflows/publish', { method: 'POST', headers: Object.assign({ 'Content-Type': 'application/json' }, _inceptionAuthHeaders()), body: JSON.stringify({ name: name, version: version }) }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + const data = await res.json(); + showToast('Published ' + (data.published || name + '@' + version) + ' (' + (data.tier || 'local-published') + ')'); + _wfMkt.loaded = false; + } catch (err) { + showToast('Publish failed: ' + err.message, 'error'); + } + } + + async function wfMktRefreshCatalogue() { + if (!dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')) { showToast('Owner role is required.', 'error'); return; } + _wfMkt.busy = 'refresh'; wfMktRerender(); + try { + const res = await fetch('/api/marketplace/refresh', { method: 'POST', headers: _inceptionAuthHeaders() }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + showToast('Hub refresh requested; the catalogue updates as repositories are re-ingested.'); + } catch (err) { + showToast('Refresh failed: ' + err.message, 'error'); + } + _wfMkt.busy = ''; + wfMktRerender(); + } + + // Called from renderGovWorkflows when the Marketplace view is active. + function renderWorkflowMarketplaceView(owner) { + if (!_wfMkt.loaded && !_wfMkt.loading) wfMktLoad(false); + return renderWorkflowMarketplace(_wfMkt, (typeof _workflowState !== 'undefined' && _workflowState.workflows) || [], owner); + } diff --git a/src/pkg/dashboard/static-src/manifest.txt b/src/pkg/dashboard/static-src/manifest.txt index 18e58c532f..45731eba3a 100644 --- a/src/pkg/dashboard/static-src/manifest.txt +++ b/src/pkg/dashboard/static-src/manifest.txt @@ -25,6 +25,7 @@ css/config-dialog.css css/workflows.css css/workflows-editor.css css/workflows-ingest.css +css/workflows-marketplace.css css/chat.css css/nous.css css/knowledge.css @@ -141,6 +142,7 @@ js/config-workflows.js js/config-workflows-library.js js/config-workflows-editor.js js/config-workflows-ingest-sources.js +js/config-workflows-marketplace.js js/config-watchdog-security.js js/config-features.js js/integration-execution-settings.js diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index fbaaa18122..db978b8760 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -5313,6 +5313,17 @@ .wf-src-run-succeeded { color: var(--status-ok); } .wf-src-run-failed { color: var(--status-error); } .wf-src-run-running { color: var(--status-warn); } +/* Settings → Workflows → Marketplace (#17620 phase 7): catalogue table, tier badges, stale banner */ + .wf-prov-local-published { color: var(--accent); border-color: color-mix(in srgb, var(--accent) 40%, transparent); background: color-mix(in srgb, var(--accent) 10%, transparent); } + .wf-mkt-toolbar { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-3); margin: var(--sp-4) 0; } + .wf-mkt-toolbar .wf-mkt-search { flex: 1 1 220px; min-width: 180px; font: inherit; font-size: var(--fs-sm); color: var(--text); background: var(--bg-soft); border: var(--line-width) solid var(--border); border-radius: var(--r-sm); padding: var(--sp-2) var(--sp-3); } + .wf-mkt-toolbar .wf-mkt-tier { font: inherit; font-size: var(--fs-sm); color: var(--text); background: var(--bg-soft); border: var(--line-width) solid var(--border); border-radius: var(--r-sm); padding: var(--sp-2) var(--sp-3); } + .wf-mkt-count { font-size: var(--fs-sm); white-space: nowrap; } + .wf-mkt-sub { font-size: var(--fs-xs); margin-top: var(--sp-1); } + .wf-mkt-banner { font-size: var(--fs-sm); padding: var(--sp-3) var(--sp-4); border-radius: var(--r-sm); margin: var(--sp-3) 0; border: var(--line-width) solid var(--border); } + .wf-mkt-stale { color: var(--status-warn); border-color: color-mix(in srgb, var(--status-warn) 40%, transparent); background: color-mix(in srgb, var(--status-warn) 10%, transparent); } + .wf-mkt-flag { display: inline-block; margin-top: var(--sp-1); font-size: var(--fs-xs); font-weight: 700; color: var(--status-warn); white-space: nowrap; } + .wf-mkt-table td { vertical-align: top; } /* Hive Chat Panel */ .hive-chat-fab { position: fixed; bottom: 24px; right: 24px; z-index: 9000; @@ -62855,7 +62866,7 @@

Agent Watchdog

// // All HTML builders are pure functions of their inputs so the node // harness (workflows_library_ui_test.go) can run them without a DOM. - const WF_LIB_VIEWS = ['library', 'detail', 'ingest', 'editor']; + const WF_LIB_VIEWS = ['library', 'detail', 'ingest', 'editor', 'marketplace']; const WF_PROVENANCE_LABELS = { 'built-in': 'built-in', 'curated': 'curated', @@ -63119,7 +63130,7 @@

Agent Watchdog

} function wfLibNav(view) { - const items = [['library', 'Library'], ['ingest', 'Ingest'], ['editor', 'Editor']]; + const items = [['library', 'Library'], ['ingest', 'Ingest'], ['editor', 'Editor'], ['marketplace', 'Marketplace']]; return '
' + items.map(([id, label]) => `` ).join('') + '
'; @@ -63144,6 +63155,7 @@

Agent Watchdog

: ``) : '', owner ? `` : '', ``, + typeof wfMktPublishButton === 'function' ? wfMktPublishButton(e, owner) : '', canDelete ? `` : (owner && !builtin && bound.length ? `bound` : ''), ].join(''); @@ -63314,6 +63326,7 @@

Agent Watchdog

: ``) : '', owner ? `` : '', ``, + typeof wfMktPublishButton === 'function' ? wfMktPublishButton(e, owner) : '', owner && !builtin && !bound.length ? `` : '', ].join(''); return '
' + @@ -63623,6 +63636,8 @@

Agent Watchdog

body = renderWorkflowIngest(_wfLib.ingest, owner); } else if (_wfLib.view === 'editor') { body = renderWorkflowEditor(owner); + } else if (_wfLib.view === 'marketplace') { + body = renderWorkflowMarketplaceView(owner); } else { body = renderWorkflowLibraryTable(_workflowState.workflows, _workflowState.bindings, owner); } @@ -65809,6 +65824,210 @@

Agent Watchdog

wfLibRerender(); } } +// Settings → Workflows → Marketplace (hivecommons/hive#17620, RFC phase 7). + // + // Browses the hub's workflow-model catalogue through the spoke proxy + // (GET /api/marketplace/workflows, ETag-cached server-side), imports an + // entry into the local library (the spoke re-verifies the bundle and + // recompiles the model; the imported model is pinned name@version and + // no binding moves), and lets an owner publish a local model back to + // the hub. Catalogue tiers: built-in · curated · ingested · unreviewed · + // local-published. Ratings/adoption columns are reserved and render + // "—" until the hub fills them. + // + // Render functions are pure (node-testable in + // workflows_marketplace_ui_test.go); state lives in _wfMkt. + const WF_MKT_TIERS = ['built-in', 'curated', 'ingested-unreviewed', 'local-published']; + const WF_MKT_TIER_LABELS = { + 'built-in': 'built-in', + 'curated': 'curated', + 'ingested-unreviewed': 'ingested · unreviewed', + 'local-published': 'local-published', + }; + const WF_MKT_TIER_TIPS = { + 'built-in': 'Ships with hive; identical on every spoke.', + 'curated': 'Reviewed and vouched for by a hub maintainer. Never overwritten by ingest; a newer ingest only sets a flag.', + 'ingested-unreviewed': 'Derived nightly from a CNCF project\'s pipeline files; nobody has reviewed it. Compiles with warnings.', + 'local-published': 'Published by another hive from its local library. Verified by the hub, not reviewed.', + }; + const _wfMkt = { loaded: false, loading: false, index: null, stale: false, error: '', fetchedAt: '', search: '', tier: '', busy: '' }; + + function wfMktTierBadge(tier) { + const t = WF_MKT_TIER_LABELS[tier] ? tier : 'local-published'; + return `${esc(WF_MKT_TIER_LABELS[t])}`; + } + + // Reserved read-only cells: "—" until the hub populates the object. + function wfMktReservedCell(obj, key, suffix) { + if (!obj || typeof obj !== 'object' || obj[key] === undefined || obj[key] === null || obj[key] === '') return '—'; + return esc(String(obj[key])) + (suffix ? esc(suffix) : ''); + } + + // Case-insensitive match on name, description and source; tier filter + // exact. Pure. + function wfMktFilter(entries, search, tier) { + const q = String(search || '').trim().toLowerCase(); + return (Array.isArray(entries) ? entries : []).filter(e => { + if (!e) return false; + if (tier && e.tier !== tier) return false; + if (!q) return true; + return [e.name, e.version, e.description, e.source, e.publishedBy].some(v => String(v || '').toLowerCase().includes(q)); + }); + } + + // Is name@version already in the local library? Imported models keep + // the catalogue's version when free, so a match means "installed". + function wfMktInstalled(entry, workflows) { + return (Array.isArray(workflows) ? workflows : []).some(w => w && w.name === entry.name && w.version === entry.version); + } + + function renderWorkflowMarketplaceRows(entries, workflows, owner, busyKey) { + const list = Array.isArray(entries) ? entries.slice() : []; + list.sort((a, b) => { + const ta = WF_MKT_TIERS.indexOf(a.tier), tb = WF_MKT_TIERS.indexOf(b.tier); + if (ta !== tb) return (ta < 0 ? 99 : ta) - (tb < 0 ? 99 : tb); + return (a.name + '@' + a.version).localeCompare(b.name + '@' + b.version); + }); + if (!list.length) return 'No catalogue entries match.'; + return list.map(e => { + const key = e.name + '@' + e.version; + const installed = wfMktInstalled(e, workflows); + const busy = busyKey === key; + let action; + if (installed) action = ``; + else if (owner) action = ``; + else action = '—'; + return `` + + `${esc(key)}${e.description ? `
${esc(e.description)}
` : ''}` + + `${wfMktTierBadge(e.tier)}${e.newerIngestAvailable ? '
newer ingest available
' : ''}` + + `${esc(e.source || '')}${e.sourceSHA ? `
${esc(String(e.sourceSHA).slice(0, 12))}
` : ''}` + + `${esc(String(e.states || 0))} / ${esc(String(e.columns || 0))}` + + `${e.publishedAt ? esc(formatDashboardDateTime(e.publishedAt)) : '—'}${e.publishedBy ? `
${esc(e.publishedBy)}
` : ''}` + + `${wfMktReservedCell(e.adoption, 'hives')}` + + `${wfMktReservedCell(e.ratings, 'average')}` + + `${installed ? 'in library' : '—'}` + + `${action}`; + }).join(''); + } + + function renderWorkflowMarketplace(state, workflows, owner) { + const st = state || {}; + const entries = (st.index && Array.isArray(st.index.entries)) ? st.index.entries : []; + const shown = wfMktFilter(entries, st.search, st.tier); + const counts = {}; + entries.forEach(e => { counts[e.tier] = (counts[e.tier] || 0) + 1; }); + let banner = ''; + if (st.stale) banner = `
Showing the last catalogue this hive cached${st.fetchedAt ? ' (' + esc(formatDashboardDateTime(st.fetchedAt)) + ')' : ''}: the hub could not be reached${st.error ? ' — ' + esc(st.error) : ''}.
`; + else if (st.error) banner = ``; + const tierOpts = [''].concat(WF_MKT_TIERS.map(t => + ``)).join(''); + const toolbar = '
' + + `` + + `` + + `${shown.length} of ${entries.length}` + + `` + + (owner ? `` : '') + + '
'; + let body; + if (!st.loaded && st.loading) body = '

Loading the catalogue…

'; + else body = '' + + '' + renderWorkflowMarketplaceRows(shown, workflows, owner, st.busy) + '
ModelTierSourceS / CPublishedAdoptionRatingLocal
'; + return '
' + + '

Marketplace

' + + '

Workflow models published to the hub: built-in and curated lifecycles, nightly ingests of CNCF project pipelines (unreviewed), and models other hives published. Importing verifies the bundle\'s hash, recompiles the model on this hive and saves it pinned to its catalogue version; no repository is bound and nothing moves automatically. The hub to browse is ' + yamlHint('workflow.marketplace.url') + ' (defaults to this hive\'s hub); the spoke re-checks its cached copy every ' + yamlHint('workflow.marketplace.refresh_interval') + '. Turn the catalogue off with ' + yamlHint('workflow.marketplace.enabled') + '.

' + + banner + toolbar + body + '
'; + } + + // Publish button for library rows and the detail view; '' unless the + // caller is an owner and the model is publishable (not built-in). + function wfMktPublishButton(entry, owner) { + if (!owner || !entry || wfProvenanceOf(entry) === 'built-in' || entry.compiled === false) return ''; + return ``; + } + + function wfMktRerender() { if (typeof wfLibRerender === 'function') wfLibRerender(); } + function wfMktSetSearch(value) { _wfMkt.search = String(value || ''); wfMktRerender(); } + function wfMktSetTier(value) { _wfMkt.tier = String(value || ''); wfMktRerender(); } + + async function wfMktLoad(force) { + if (_wfMkt.loading) return; + _wfMkt.loading = true; _wfMkt.error = ''; + wfMktRerender(); + try { + const res = await fetch('/api/marketplace/workflows' + (force ? '?refresh=1' : ''), { headers: _inceptionAuthHeaders() }); + let data = null; + try { data = await res.json(); } catch (e) { data = null; } + if (!res.ok && !(data && data.index)) throw new Error((data && data.error) || ('HTTP ' + res.status)); + _wfMkt.index = data.index || { entries: [] }; + _wfMkt.stale = !!data.stale; + _wfMkt.fetchedAt = data.fetchedAt || ''; + _wfMkt.error = data.error || ''; + _wfMkt.loaded = true; + } catch (err) { + _wfMkt.error = 'Could not load the catalogue: ' + err.message; + _wfMkt.loaded = true; + } + _wfMkt.loading = false; + wfMktRerender(); + } + + function wfMktReload() { wfMktLoad(true); } + + async function wfMktImport(name, version) { + if (!dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')) { showToast('Owner role is required to import.', 'error'); return; } + const key = name + '@' + version; + _wfMkt.busy = key; wfMktRerender(); + try { + const res = await fetch('/api/marketplace/workflows/' + encodeURIComponent(name) + '/' + encodeURIComponent(version) + '/import', { method: 'POST', headers: _inceptionAuthHeaders() }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + const data = await res.json(); + const e = data.entry || {}; + showToast('Imported ' + key + ' as ' + e.name + '@' + e.version + ' (verified, compiled locally)'); + _wfMkt.busy = ''; + if (typeof _workflowState !== 'undefined') _workflowState.loaded = false; + if (typeof loadWorkflowLibrary === 'function') await loadWorkflowLibrary(); + openWorkflowLibraryEntry(e.name, e.version); + } catch (err) { + _wfMkt.busy = ''; + showToast('Import failed: ' + err.message, 'error'); + } + wfMktRerender(); + } + + async function wfMktPublish(name, version) { + if (!dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')) { showToast('Owner role is required to publish.', 'error'); return; } + const ok = await hiveConfirm({ title: 'Publish ' + name + '@' + version + ' to the hub?', message: 'A signed bundle (model + compiled profile + hash) is sent to the hub marketplace under this hive\'s identity as tier local-published. Catalogue entries are immutable: once published, this name@version cannot be changed, only superseded by a new version.', confirmLabel: 'Publish' }); + if (!ok) return; + try { + const res = await fetch('/api/marketplace/workflows/publish', { method: 'POST', headers: Object.assign({ 'Content-Type': 'application/json' }, _inceptionAuthHeaders()), body: JSON.stringify({ name: name, version: version }) }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + const data = await res.json(); + showToast('Published ' + (data.published || name + '@' + version) + ' (' + (data.tier || 'local-published') + ')'); + _wfMkt.loaded = false; + } catch (err) { + showToast('Publish failed: ' + err.message, 'error'); + } + } + + async function wfMktRefreshCatalogue() { + if (!dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')) { showToast('Owner role is required.', 'error'); return; } + _wfMkt.busy = 'refresh'; wfMktRerender(); + try { + const res = await fetch('/api/marketplace/refresh', { method: 'POST', headers: _inceptionAuthHeaders() }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + showToast('Hub refresh requested; the catalogue updates as repositories are re-ingested.'); + } catch (err) { + showToast('Refresh failed: ' + err.message, 'error'); + } + _wfMkt.busy = ''; + wfMktRerender(); + } + + // Called from renderGovWorkflows when the Marketplace view is active. + function renderWorkflowMarketplaceView(owner) { + if (!_wfMkt.loaded && !_wfMkt.loading) wfMktLoad(false); + return renderWorkflowMarketplace(_wfMkt, (typeof _workflowState !== 'undefined' && _workflowState.workflows) || [], owner); + } // ── Watchdog activity strip (#7254) ────────────────────────────────────── // "Review the watchdog-*-observed entries in the Audit Log, then switch to // Heal" is only a data-driven decision if the data is on the page. The diff --git a/src/pkg/dashboard/workflows_marketplace_ui_test.go b/src/pkg/dashboard/workflows_marketplace_ui_test.go new file mode 100644 index 0000000000..c8cf1cc14a --- /dev/null +++ b/src/pkg/dashboard/workflows_marketplace_ui_test.go @@ -0,0 +1,127 @@ +package dashboard + +import ( + "os/exec" + "strings" + "testing" +) + +// TestWorkflowsMarketplaceTabWiring pins the RFC phase-7 surface +// (hivecommons/hive#17620): the Workflows nav grows a Marketplace item, +// the governor dispatcher routes to the marketplace view, import/publish/ +// refresh go through the spoke proxy routes, and library rows/detail carry +// a Publish button. +func TestWorkflowsMarketplaceTabWiring(t *testing.T) { + html := indexHTML(t) + for _, want := range []string{ + `['marketplace', 'Marketplace']`, + `const WF_LIB_VIEWS = ['library', 'detail', 'ingest', 'marketplace'];`, + `} else if (_wfLib.view === 'marketplace') {`, + `body = renderWorkflowMarketplaceView(owner);`, + `function renderWorkflowMarketplace(state, workflows, owner)`, + `function renderWorkflowMarketplaceRows(entries, workflows, owner, busyKey)`, + `function wfMktPublishButton(entry, owner)`, + `wfMktPublishButton(e, owner)`, + `fetch('/api/marketplace/workflows' + (force ? '?refresh=1' : '')`, + `'/import', { method: 'POST'`, + `fetch('/api/marketplace/workflows/publish', { method: 'POST'`, + `fetch('/api/marketplace/refresh', { method: 'POST'`, + `await hiveConfirm({ title: 'Publish '`, + `yamlHint('workflow.marketplace.enabled')`, + `.wf-mkt-stale {`, + `.wf-prov-local-published {`, + } { + if !strings.Contains(html, want) { + t.Errorf("index.html missing Marketplace wiring %q", want) + } + } + mkt := html[strings.Index(html, "function wfMktTierBadge("):] + mkt = mkt[:strings.Index(mkt, "function renderWorkflowMarketplaceView(")] + for _, banned := range []string{"toLocaleString", "toLocaleDateString", "download="} { + if strings.Contains(mkt, banned) { + t.Errorf("marketplace JS must not use %q", banned) + } + } +} + +// Pure helpers under node: tier badge fallback, search/tier filtering, +// installed detection, row ordering by tier, the newer-ingest flag, the +// reserved adoption/rating cells, the stale banner and the publish button +// gating. +func TestWorkflowsMarketplaceNode(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node not installed") + } + html := indexHTML(t) + script := `const assert = require('assert'); +function esc(s) { return String(s).replace(/&/g, '&').replace(/' + p + ''; } +const WF_MKT_TIERS = ['built-in', 'curated', 'ingested-unreviewed', 'local-published']; +const WF_MKT_TIER_LABELS = { 'built-in': 'built-in', 'curated': 'curated', 'ingested-unreviewed': 'ingested · unreviewed', 'local-published': 'local-published' }; +const WF_MKT_TIER_TIPS = { 'built-in': 'b', 'curated': 'c', 'ingested-unreviewed': 'i', 'local-published': 'l' }; +` + jsFunc(t, html, "wfProvenanceOf") + jsFunc(t, html, "wfMktTierBadge") + jsFunc(t, html, "wfMktReservedCell") + + jsFunc(t, html, "wfMktFilter") + jsFunc(t, html, "wfMktInstalled") + jsFunc(t, html, "renderWorkflowMarketplaceRows") + + jsFunc(t, html, "renderWorkflowMarketplace") + jsFunc(t, html, "wfMktPublishButton") + ` +assert.ok(wfMktTierBadge('curated').includes('wf-prov-curated')); +assert.ok(wfMktTierBadge('bogus').includes('wf-prov-local-published'), 'unknown tier falls back to local-published'); +assert.strictEqual(wfMktReservedCell({}, 'hives'), '—'); +assert.strictEqual(wfMktReservedCell(null, 'hives'), '—'); +assert.strictEqual(wfMktReservedCell({ hives: 12 }, 'hives'), '12'); + +const entries = [ + { name: 'envoy', version: 'ingested-20260314.1', tier: 'ingested-unreviewed', description: 'Envoy release train', source: 'envoyproxy/envoy', states: 9, columns: 7, publishedAt: '2026-03-14T03:00:00Z', publishedBy: 'marketplace-refresh', sourceSHA: 'abcdef1234567890', adoption: {}, ratings: {}, benchmarks: {} }, + { name: 'techdocs', version: '2', tier: 'curated', description: 'Docs engagement', source: 'cncf/techdocs', states: 9, columns: 5, publishedAt: '2026-03-01T00:00:00Z', publishedBy: 'root', newerIngestAvailable: true, adoption: { hives: 3 }, ratings: {} }, + { name: 'hive-default', version: '1', tier: 'built-in', states: 8, columns: 7 }, + { name: 'mine', version: '1', tier: 'local-published', source: 'acme/mine', publishedBy: 'hive:acme' }, +]; +assert.strictEqual(wfMktFilter(entries, '', '').length, 4); +assert.deepStrictEqual(wfMktFilter(entries, 'ENVOY', '').map(e => e.name), ['envoy'], 'search is case-insensitive and matches source'); +assert.deepStrictEqual(wfMktFilter(entries, '', 'curated').map(e => e.name), ['techdocs']); +assert.deepStrictEqual(wfMktFilter(entries, 'docs', 'ingested-unreviewed'), []); +assert.deepStrictEqual(wfMktFilter(null, 'x', ''), []); + +const local = [{ name: 'techdocs', version: '2' }, { name: 'hive-default', version: '1', readOnly: true }]; +assert.strictEqual(wfMktInstalled(entries[1], local), true); +assert.strictEqual(wfMktInstalled(entries[0], local), false); + +const rows = renderWorkflowMarketplaceRows(entries, local, true, 'envoy@ingested-20260314.1'); +const keys = [...rows.matchAll(/data-wf-mkt="([^"]+)"/g)].map(m => m[1]); +assert.deepStrictEqual(keys, ['hive-default@1', 'techdocs@2', 'envoy@ingested-20260314.1', 'mine@1'], 'built-in, curated, ingested, local-published'); +assert.ok(rows.includes('newer ingest available'), 'curated with later ingest is flagged'); +assert.ok(rows.includes('Importing…'), 'busy key disables its import button'); +assert.ok(rows.includes('data-action="openWorkflowLibraryEntry" data-arg0="techdocs" data-arg1="2"'), 'installed entries open the library'); +assert.ok(rows.includes('>in library<')); +assert.ok(rows.includes('data-action="wfMktImport" data-arg0="mine" data-arg1="1"')); +assert.ok(rows.includes('>3<'), 'adoption.hives renders when present'); +assert.ok(rows.includes('abcdef123456
'), 'source SHA is abbreviated'); +assert.ok(!rows.includes('toLocaleString')); +const readerRows = renderWorkflowMarketplaceRows(entries, [], false, ''); +assert.ok(!readerRows.includes('wfMktImport'), 'readers get no import button'); +assert.ok(renderWorkflowMarketplaceRows([], [], true, '').includes('No catalogue entries match.')); + +const view = renderWorkflowMarketplace({ loaded: true, loading: false, index: { entries }, stale: true, error: 'dial tcp: refused', fetchedAt: '2026-03-14T04:00:00Z', search: 'tech', tier: '' }, local, true); +assert.ok(view.includes('wf-mkt-stale'), 'stale banner'); +assert.ok(view.includes('dial tcp: refused')); +assert.ok(view.includes('1 of 4'), 'search narrows the count'); +assert.ok(view.includes('data-action="wfMktRefreshCatalogue"'), 'owner sees refresh'); +assert.ok(view.includes('curated (1)'), 'tier filter shows counts'); +const readerView = renderWorkflowMarketplace({ loaded: true, index: { entries }, error: 'boom' }, [], false); +assert.ok(!readerView.includes('wfMktRefreshCatalogue')); +assert.ok(readerView.includes('wf-ingest-error') && readerView.includes('boom'), 'non-stale error is an alert'); +assert.ok(renderWorkflowMarketplace({ loaded: false, loading: true }, [], true).includes('Loading the catalogue…')); + +assert.strictEqual(wfMktPublishButton({ name: 'hive-default', version: '1', readOnly: true }, true), '', 'built-in is not publishable'); +assert.strictEqual(wfMktPublishButton({ name: 'x', version: '1' }, false), '', 'readers cannot publish'); +assert.strictEqual(wfMktPublishButton({ name: 'x', version: '1', compiled: false }, true), '', 'uncompilable is not publishable'); +assert.ok(wfMktPublishButton({ name: 'x', version: '1', provenance: 'local' }, true).includes('data-action="wfMktPublish" data-arg0="x" data-arg1="1"')); +console.log('ok'); +` + out, err := runNodeStdin(node, script) + if err != nil { + t.Fatalf("node: %v\n%s", err, out) + } + if !strings.Contains(string(out), "ok") { + t.Fatalf("unexpected node output: %s", out) + } +} From 81350006138beff5561a1650a5d1a8e5c58b8b2d Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 11 Nov 2026 10:06:41 -0700 Subject: [PATCH 6/9] =?UTF-8?q?=F0=9F=93=96=20workflow=20models:=20documen?= =?UTF-8?q?t=20bundles,=20hub=20marketplace=20and=20spoke=20import/publish?= =?UTF-8?q?=20(RFC=20phase=207)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --- .../added-workflowmodel-marketplace.md | 1 + src/docs/design/workflow-models.md | 127 ++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 changelog.d/added-workflowmodel-marketplace.md diff --git a/changelog.d/added-workflowmodel-marketplace.md b/changelog.d/added-workflowmodel-marketplace.md new file mode 100644 index 0000000000..bae3f3071c --- /dev/null +++ b/changelog.d/added-workflowmodel-marketplace.md @@ -0,0 +1 @@ +- **workflow models:** RFC phase 7 — signed bundles (`pkg/workflowmodel/bundle`: model + compiled profile + sha256; import always re-verifies and recompiles locally), a hub marketplace catalogue (`/api/marketplace/*`: immutable `name@version` entries, ETag index, tiers built-in · curated · ingested-unreviewed · local-published, nightly/on-demand refresh of the embedded CNCF corpus publishing `ingested-YYYYMMDD.N`, curated versions only flagged `newerIngestAvailable`), spoke import/publish/proxy routes with `workflow.marketplace.{enabled,url,refresh_interval}`, and a Settings → Workflows → Marketplace view with Import, Publish and Refresh catalogue ([#17620](https://github.com/hivecommons/hive/issues/17620)) diff --git a/src/docs/design/workflow-models.md b/src/docs/design/workflow-models.md index 326f491425..d5e7bd53e5 100644 --- a/src/docs/design/workflow-models.md +++ b/src/docs/design/workflow-models.md @@ -679,3 +679,130 @@ from the capabilities route's list or a text field, namespace, selector, all-nam observed-run list) and *From a written spec* (paste, URL or repository file; a *Use AI assistance* checkbox rendered only when the spoke reports a gateway; the LLM note and section list in the result). Both reuse the repository ingest's result block, diagnostics and diagram. + +## Bundles and marketplace (RFC phase 7) + +Phase 7 lets a model leave the hive it was authored on. Three pieces: a signed +**bundle** format (`pkg/workflowmodel/bundle`), an immutable **catalogue** on the +hub (`pkg/hub/marketplace*.go`), and the spoke's **import / publish / browse** +surface (`pkg/dashboard/api_workflows_marketplace.go`, Settings → Workflows → +Marketplace). + +### Bundle — `pkg/workflowmodel/bundle` + +``` +{ apiVersion: "workflow.hivecommons.dev/v1alpha1-bundle", kind: "WorkflowBundle", + model: , profile: , hash: , createdAt, createdBy, tier } +``` + +- `Export(m, by, now)` compiles the model and refuses on compile errors; the hash is + `CanonicalHash(m)` — sha256 of the model's canonical JSON (sorted keys, no + insignificant whitespace, `apiVersion/kind/metadata/spec` only). +- `Encode`/`Decode` are canonical and strict (`DisallowUnknownFields`); the bundle + bytes themselves are deterministic so a store can compare them byte-for-byte. +- `Verify(b)` recomputes the hash **and recompiles the model locally**, comparing the + result structurally with the published profile. A mismatch is + `ErrProfileMismatch` ("published profile does not match local compile"). Nothing + downstream ever trusts a published profile: the receiving side always recompiles. +- Tiers are provenance for the catalogue: `built-in`, `curated`, + `ingested-unreviewed`, `local`, `local-published`. `Export` derives the default tier + from `spec.meta.provenance`. +- `IndexEntry`/`Index` are the shared wire types for the catalogue index (below) so the + hub and spoke agree without the hub importing dashboard code. + +### Hub catalogue — `pkg/hub/marketplace*.go` + +Storage is a directory under the hub data dir: + +``` +/data/marketplace/ + workflows//.bundle.json immutable; refuse overwrite of name@version + index.json derived; rebuilt from the bundles if missing + refresh/__.json last ingest run per repository +``` + +Entries are keyed `name@version` plus hash. Publishing the identical bundle again is a +no-op (`200`); different content under an existing key is `409`. The index carries +`{name, version, hash, tier, description, source, states, columns, publishedAt, +publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}`; +the last three are reserved and empty today. `ETag` is the sha256 of the index bytes. + +Routes (see [hub-server.md](../api-reference/hub-server.md)): + +| Route | Who | Notes | +|---|---|---| +| `GET /api/marketplace/workflows` | public | index; `If-None-Match` → `304` | +| `GET /api/marketplace/workflows/{name}/{version}/bundle` | public | one bundle, `Cache-Control: immutable` | +| `POST /api/marketplace/workflows` | spoke bearer or hub admin | publish; hub re-verifies | +| `POST /api/marketplace/refresh[?repo=]` | hub admin | on-demand ingest | +| `GET /api/marketplace/status` | public | last refresh, counts by tier | + +**Who may publish what.** A spoke authenticates with its heartbeat bearer bound to the +`X-Hive-ID` header (the same per-hive key the heartbeat and NPS lanes verify) and may +assert `local`/`local-published` (normalised to `local-published`) or +`ingested-unreviewed`. `curated` and `built-in` require a hub-admin session with +same-origin CSRF evidence. The hub never stores a bundle it did not itself `Verify`. + +**Refresh.** The corpus is the CNCF survey list, now embedded as +`pkg/workflowmodel/ingest/corpus` (262 projects / 299 repositories; `cmd/workflow-survey` +falls back to it when no `projects.json` is given) plus +`HIVE_HUB_MARKETPLACE_EXTRA_REPOS`. For each repository the hub fetches the default-branch +tarball through its GitHub identity, keeps only pipeline files (`ingest.ReadTree`), and +records `{sourceSHA, filesHash, files, checkedAt}`. Unchanged hash → no-op. Changed → +`ingest.Lifecycle` → `bundle.Export` → publish as `ingested-YYYYMMDD.N`, tier +`ingested-unreviewed`, with the source SHA in an annotation. Curated versions are never +touched; `marketplaceNormaliseIndex` flags a curated entry `newerIngestAvailable` when a +later ingest of the same name exists. The crawl is sequential with a short pause between +repositories and honours `X-RateLimit-Remaining`/`Reset`. It runs nightly at +`HIVE_HUB_MARKETPLACE_REFRESH_SCHEDULE` (HH:MM UTC, default `03:00`, empty disables), on +demand, and once at hub start when the catalogue is empty and a GitHub token is present +(in a goroutine; startup is not blocked). `HIVE_HUB_MARKETPLACE_DISABLED` turns the +routes off (`404`). + +### Spoke — import, publish, browse + +Config (`hive.yaml`): + +```yaml +workflow: + marketplace: + enabled: true # default true; false hides the tab and 404s the routes + url: "" # default: hub.url + refresh_interval: 15m # how long the spoke trusts its cached index before revalidating +``` + +Routes (`pkg/dashboard/api_workflows_marketplace.go`): + +- `GET /api/workflows/{name}/{version}/bundle` — export a stored model as a bundle (any role). +- `POST /api/workflows/import` (owner) — body is a bundle; `Verify` → recompile → `store.Save` + pinned to the bundle's version when free, else the next free version; `201 {entry, + verified:true}`; `422` when verification fails. Audit `workflow_import`. +- `GET /api/marketplace/workflows` — proxy to the hub with `If-None-Match`; response + `{index, stale, etag, fetchedAt, error?}`. Hub unreachable → `502` with the cached index + and `stale:true` so the UI can still render. +- `POST /api/marketplace/workflows/{name}/{version}/import` (owner) — fetch the bundle from + the hub and import as above. Audit `workflow_import`. +- `POST /api/marketplace/workflows/publish {name, version}` (owner) — export and POST to + the hub with the spoke's heartbeat credential; `201 {published, hash, tier}`. Audit + `workflow_publish`. +- `POST /api/marketplace/refresh[?repo=]` (owner) — forwards to the hub; a non-admin hive + surfaces the hub's `403`. Audit `workflow_marketplace_refresh`. + +Imported models are pinned `name@version`; a newer catalogue version is a new library entry +and **bindings never move automatically** — rebinding stays an explicit act on Settings → +Repos. Built-in names cannot be imported over. + +UI: Settings → Workflows → **Marketplace** (`config-workflows-marketplace.js`) lists the +catalogue with tier badges, search, a tier filter, the `newer ingest available` flag, +reserved Adoption/Rating columns (rendered `—`), **Import** (→ verified, compiled, library +entry opened), **Reload**, and for owners **Refresh catalogue**; library rows and the detail +view gain **Publish** (confirmed via `hiveConfirm`). A stale banner shows when the proxy +served cached data. + +### Known gaps (phase 7) + +- `adoption`, `ratings`, `benchmarks` are reserved and empty; nothing collects them yet. +- Hub-side marketplace settings are environment variables, not hub YAML. +- The hub logs publishes/refreshes through its structured logger; it has no audit store. +- Ingest versions are per-day sequence numbers, not semantic; curation (promoting an ingest + to `curated`) is a manual admin publish of a reviewed model. From 1a7ffd82792711bf0fa49db6490ec50a0940951a Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 11 Nov 2026 11:13:35 -0700 Subject: [PATCH 7/9] =?UTF-8?q?=F0=9F=8C=B1=20marketplace=20tests:=20drop?= =?UTF-8?q?=20fixed=20sleeps=20and=20unused=20field?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the two time.Sleep polls in the hub marketplace tests with testutil.Eventually / direct cancel so the sleep ratchet stays at its baseline, and remove the unused fakeHub.down field flagged by golangci-lint. Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --- .../api_workflows_marketplace_test.go | 1 - src/pkg/hub/marketplace_test.go | 20 ++++++------------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/pkg/dashboard/api_workflows_marketplace_test.go b/src/pkg/dashboard/api_workflows_marketplace_test.go index 4a5add4cb1..875fe0b7d0 100644 --- a/src/pkg/dashboard/api_workflows_marketplace_test.go +++ b/src/pkg/dashboard/api_workflows_marketplace_test.go @@ -68,7 +68,6 @@ type fakeHub struct { lastHive string refreshes atomic.Int32 publishRC int - down bool } func newFakeHub(t *testing.T) (*fakeHub, *httptest.Server) { diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index 17eee4532e..5a0230dacf 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -16,6 +16,7 @@ import ( "testing" "time" + "github.com/hivecommons/hive/internal/testutil" "github.com/hivecommons/hive/pkg/workflowmodel" "github.com/hivecommons/hive/pkg/workflowmodel/bundle" ) @@ -570,20 +571,11 @@ func TestMarketplaceRefreshRoutesAndStatus(t *testing.T) { if rec = mktDo(s, admin("/api/marketplace/refresh")); rec.Code != 202 { t.Fatalf("full: %d %s", rec.Code, rec.Body.String()) } - deadline := time.Now().Add(5 * time.Second) - for { + testutil.Eventually(t, 5*time.Second, func() bool { rf.mu.Lock() - last := rf.last - running := rf.running - rf.mu.Unlock() - if !running && last != nil && strings.HasPrefix(last.Trigger, "on-demand:") { - break - } - if time.Now().After(deadline) { - t.Fatal("background refresh did not finish") - } - time.Sleep(10 * time.Millisecond) - } + defer rf.mu.Unlock() + return !rf.running && rf.last != nil && strings.HasPrefix(rf.last.Trigger, "on-demand:") + }, "background refresh did not finish") rec = mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/status", nil)) if rec.Code != 200 { @@ -697,7 +689,7 @@ func TestMarketplaceStartRefreshDisabledAndSeedGate(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) done = make(chan struct{}) go func() { s.StartMarketplaceRefresh(ctx); close(done) }() - time.Sleep(20 * time.Millisecond) + // Cancelling before the loop parks is fine: ctx.Done is observed either way. cancel() select { case <-done: From 65ff2a52de54de3e6affc5f1ed17fe265399cd30 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 11 Nov 2026 11:50:03 -0700 Subject: [PATCH 8/9] =?UTF-8?q?=F0=9F=93=96=20hub=20api-reference:=20refre?= =?UTF-8?q?sh=20server.go=20line=20citations=20after=20rebase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --- src/docs/api-reference/hub-server.md | 58 ++++++++++++++-------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index 65dd86f3f6..5fd503f013 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -35,42 +35,42 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/benchmarks/ingest` | Per-hive heartbeat bearer | Spoke push target for anonymized [benchmark records](../benchmarks.md): body `{hive_id, records[]}`; the hub recomputes the hive hash from the authenticated id, rejects records whose `source.hive_id` differs (`401` bad bearer, `403` unregistered hive, `400` malformed), merges accepted records by id into `/data/benchmarks//.json` and returns `202 {accepted, rejected, files}` | `pkg/hub/benchmarks.go` | | `GET` | `/api/hub/benchmarks` | Public | Benchmark publisher status: `last_ingest_at`, `last_publish_at`, `last_publish_files`, `last_error`, `pending_files`, `hive_count`, `record_count` and per-hive-hash `hives{}` counts; the hub commits pending month files hourly to `hivecommons/benchmarks` (`HIVE_BENCHMARKS_REPO`) via its App identity | `pkg/hub/benchmarks.go` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1783` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1784` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1789` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1790` | | `GET` | `/activity` | Signed-in user | Fleet activity page ([#16063](https://github.com/hivecommons/hive/issues/16063)): read-only, data-free shell that polls `/api/hub/activity` every 15 s with `If-None-Match`. No session → `307` to `/login`; blocked user → `403`. Own Content-Security-Policy (scripts and styles from `'self'` only, `script-src-attr`/`style-src-attr` `'none'`), `Cache-Control: no-cache` + content `ETag`. See [Hub Fleet activity page](../hub-fleet-activity.md) | `pkg/hub/activity_page.go` | | `GET` | `/activity/public` | Public | Public-safe Fleet activity page ([#15966](https://github.com/hivecommons/hive/issues/15966)): the same data-free shell as `/activity`, served without a session; on this path the page polls `/api/hub/activity/public` with `credentials: 'omit'`. Source of the `twitch.tv/hivecommonsfleet` broadcast. Same CSP and cache headers as `/activity`. See [Hub Fleet activity page](../hub-fleet-activity.md#public-safe-view) | `pkg/hub/activity_public.go` | | `GET` | `/activity/assets/{name}` | Public | The shared tracker assets the Fleet activity page loads, byte-identical: the row asset (`pkg/webassets/trackerrow`, [#16062](https://github.com/hivecommons/hive/issues/16062)) and the `am.*` URL-view and display-playlist helpers (`pkg/webassets/trackerview`, [#16064](https://github.com/hivecommons/hive/issues/16064)); only names in `trackerrow.Files` or `trackerview.Files`, else `404`. `Cache-Control: no-cache` + content `ETag` | `pkg/hub/activity_page.go` | -| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1715` | -| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1716` | -| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1717` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1718` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1719` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1730` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1731` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1732` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1733` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1721` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1722` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1723` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1724` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1725` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1736` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1737` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1738` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1739` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | | `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — `truncated` and `events_dropped`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, a response-level `truncated` (cap 2000), and `events[]` ([#15964](https://github.com/hivecommons/hive/issues/15964): `hive_id`, `received_at`, `seq`, `type` `opened`/`stage`/`merged`, `repo`, `n`, `kind`, `title`, `from`, `stage`, `at`, `private`), oldest first, from a fleet-wide in-memory ring of at most 1,000 events, none held longer than 24 hours (ageing out is not counted in `events_dropped`; when full, the oldest event of the hive holding the most events is dropped, counted in that hive's `events_dropped`, so one hive cannot evict the others' events). Events are filtered per user and per hive exactly like items, before aggregation, so an event for a hive the viewer cannot see is never delivered; `mod`/`author_kind` filters keep only events whose item is still on the hive's board. A spoke block carries at most 200 events; drops are disclosed per hive as `events_dropped`. New events change the `ETag`. Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | | `GET` | `/api/hub/activity/public` | Public | Public-safe activity view ([#15966](https://github.com/hivecommons/hive/issues/15966)): the `/api/hub/activity` shape for anyone, never reading a cookie, token or `Authorization` header (credentials do not change the response). Only hives the registry lists as public on github.com, and only items and events not marked `private`; `totals`, per-hive `truncated` and `events_dropped`, hive owners and per-user grants are omitted. Same filters (`400` on an invalid one). The body is built at most once per 2 s per parsed filter and shared by every anonymous caller, so rows, `generated_at` and the `ETag` change at most every 2 s; query strings that parse to the default view share one entry. Toggling a hive's visibility drops the cache at once. `ETag` + `If-None-Match` → `304`; `Cache-Control: no-cache` | `pkg/hub/activity_public.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1741` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1751` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1752` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1753` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1754` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1755` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1756` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1757` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1758` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1761` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1764` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1765` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1772` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1773` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1774` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1791` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1796` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1798` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1747` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1757` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1758` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1759` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1760` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1761` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1762` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1763` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1764` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1767` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1770` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1771` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1778` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1779` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1780` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1797` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1802` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1804` | | `GET` | `/api/marketplace/workflows` | Public | Workflow-model marketplace index ([#17620](https://github.com/hivecommons/hive/issues/17620) phase 7): `{generatedAt, entries:[{name, version, hash, tier, description, source, states, columns, publishedAt, publishedBy, sourceSHA?, newerIngestAvailable, adoption:{}, ratings:{}, benchmarks:{}}]}`. Strong `ETag` (sha256 of the body) + `Cache-Control: no-cache`; `If-None-Match` → `304`. Tiers: `built-in`, `curated`, `ingested-unreviewed`, `local-published`. `404` when `HIVE_HUB_MARKETPLACE_DISABLED` is set | `pkg/hub/marketplace.go:34` | | `GET` | `/api/marketplace/workflows/{name}/{version}/bundle` | Public | One immutable signed bundle (`{apiVersion, model, profile, hash, createdAt, createdBy, tier}`), `Cache-Control: immutable`; `404` unknown, `400` malformed key | `pkg/hub/marketplace.go:35` | | `POST` | `/api/marketplace/workflows` | Spoke heartbeat bearer + `X-Hive-ID`, or Hub admin session | Publish a bundle. Body = bundle (≤ 4 MiB); the hub re-verifies the hash and recompiles the model before storing. Publisher may assert tier `local`/`local-published` (normalised to `local-published`) or `ingested-unreviewed`; `curated`/`built-in` require a hub admin session (`403` otherwise). `201 {published, created:true, entry}`; identical republish `200 created:false`; different content under an existing `name@version` `409`; verification failure `422`; unknown hive `403`; no hub secret `503` | `pkg/hub/marketplace.go:36` | From a305b6020bf7d3ded8b6490ea5ab6c65d1645417 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 11 Nov 2026 12:07:36 -0700 Subject: [PATCH 9/9] =?UTF-8?q?=F0=9F=8C=B1=20marketplace=20UI=20test:=20l?= =?UTF-8?q?oosen=20WF=5FLIB=5FVIEWS=20assertion=20to=20coexist=20with=20ed?= =?UTF-8?q?itor=20view?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs hivecommons/hive#17620 Signed-off-by: Andrew Anderson --- src/pkg/dashboard/workflows_marketplace_ui_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/dashboard/workflows_marketplace_ui_test.go b/src/pkg/dashboard/workflows_marketplace_ui_test.go index c8cf1cc14a..7115110459 100644 --- a/src/pkg/dashboard/workflows_marketplace_ui_test.go +++ b/src/pkg/dashboard/workflows_marketplace_ui_test.go @@ -15,7 +15,7 @@ func TestWorkflowsMarketplaceTabWiring(t *testing.T) { html := indexHTML(t) for _, want := range []string{ `['marketplace', 'Marketplace']`, - `const WF_LIB_VIEWS = ['library', 'detail', 'ingest', 'marketplace'];`, + `'editor', 'marketplace'];`, `} else if (_wfLib.view === 'marketplace') {`, `body = renderWorkflowMarketplaceView(owner);`, `function renderWorkflowMarketplace(state, workflows, owner)`,