From caaf0d04f98ccfd1a69c3e3df37e67ab3d90b269 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Tue, 10 Nov 2026 22:46:33 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20workflowmodel:=20ingest=20=E2=86=92?= =?UTF-8?q?=20lifecycle=20models,=20/api/workflows/ingest,=20survey=20comp?= =?UTF-8?q?ile=20stats=20(RFC=20#17620=20P3a)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ingest.Lifecycle composes hive-default's compiled lifecycle with a repository's pipelines: PR-verify pipelines on the CI / Review states with citations, the post-merge chain as inferred columns after Merged, provenance ingested-unreviewed. New pipeline.status fact (engine reports unknown until post-merge runs are observed). ingest.ReadTree selects CI files from a checkout. POST /api/workflows/ingest fetches a repo's default branch read-only (or takes pasted files), compiles, optionally saves at the next integer version. workflow-survey compiles every corpus repo's lifecycle model: 317/317 compile, 0 refusals. Signed-off-by: Andrew Anderson --- .../added-workflowmodel-ingest-lifecycle.md | 1 + dashboard/openapi.json | 105 +++++++++++++ src/cmd/hive/main.go | 1 + src/cmd/hive/workflow_ingest.go | 37 +++++ src/cmd/hive/workflow_ingest_test.go | 16 ++ src/cmd/workflow-survey/main.go | 39 ++++- src/docs/design/workflow-models.md | 30 ++++ src/pkg/dashboard/api_workflows.go | 1 + src/pkg/dashboard/api_workflows_ingest.go | 134 ++++++++++++++++ .../dashboard/api_workflows_ingest_test.go | 105 +++++++++++++ src/pkg/dashboard/deps.go | 4 + src/pkg/stageengine/workflow.go | 3 + src/pkg/stageengine/workflow_rules_test.go | 3 + src/pkg/workflowmodel/ingest/lifecycle.go | 139 +++++++++++++++++ .../workflowmodel/ingest/lifecycle_test.go | 144 ++++++++++++++++++ src/pkg/workflowmodel/ingest/tree.go | 90 +++++++++++ src/pkg/workflowmodel/ingest/tree_test.go | 66 ++++++++ src/pkg/workflowmodel/lifecycle.go | 6 + 18 files changed, 922 insertions(+), 2 deletions(-) create mode 100644 changelog.d/added-workflowmodel-ingest-lifecycle.md create mode 100644 src/cmd/hive/workflow_ingest.go create mode 100644 src/cmd/hive/workflow_ingest_test.go create mode 100644 src/pkg/dashboard/api_workflows_ingest.go create mode 100644 src/pkg/dashboard/api_workflows_ingest_test.go create mode 100644 src/pkg/workflowmodel/ingest/lifecycle.go create mode 100644 src/pkg/workflowmodel/ingest/lifecycle_test.go create mode 100644 src/pkg/workflowmodel/ingest/tree.go create mode 100644 src/pkg/workflowmodel/ingest/tree_test.go diff --git a/changelog.d/added-workflowmodel-ingest-lifecycle.md b/changelog.d/added-workflowmodel-ingest-lifecycle.md new file mode 100644 index 0000000000..7906f340b3 --- /dev/null +++ b/changelog.d/added-workflowmodel-ingest-lifecycle.md @@ -0,0 +1 @@ +- Workflow models, RFC phase 3a ([#17620](https://github.com/hivecommons/hive/issues/17620)): an ingested repository now yields a model that compiles. `ingest.Lifecycle` composes the hive-default lifecycle (states, columns, builtin derivation, roles, gates, events) with the repository's pipelines — PR-verify pipelines bound to the CI / Review states with file citations, the post-merge chain (main CI, release, per-environment deploys, done) as new columns after Merged — every ingested element marked inferred and the model's provenance `ingested-unreviewed`. New fact `pipeline.status` names the post-merge run the chain waits on (the engine reports `unknown` until it observes post-merge runs, so nothing moves on it yet). New owner-gated `POST /api/workflows/ingest` takes `{repo}` (default branch fetched read-only through the hive's GitHub client, CI files selected by `ingest.ReadTree`) or pasted `{files}`, returns the model, compile result and diagnostics, and with `save: true` stores it at the next integer version. `cmd/workflow-survey` compiles every repository's lifecycle model and reports pass/refusal counts by code; over the 317-repository CNCF corpus 317/317 compile with 1,117 inferred-element warnings and no refusals. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 18bad9a583..22f41fe5fb 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -39356,6 +39356,79 @@ } } } + }, + "/api/workflows/ingest": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Ingest a repository's CI definitions into a lifecycle workflow model", + "description": "Reads the repository's default-branch GitHub Actions workflows and Tekton resources (or pasted files) and returns the hive-default lifecycle with the repository's pipelines bound: PR-verify pipelines on the CI / Review states, the post-merge chain as new columns. Provenance ingested-unreviewed. With save=true the model is stored as the next integer version under name. Owner role required.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "repo": { + "type": "string", + "description": "owner/name to fetch through the forge client; exactly one of repo or files" + }, + "files": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "repo-relative path to file contents, for pasted or uploaded definitions" + }, + "save": { + "type": "boolean" + }, + "name": { + "type": "string", + "description": "store name (default: slug of the repository)" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Ingested (not saved)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowIngestResult" + } + } + } + }, + "201": { + "description": "Ingested and saved", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowIngestResult" + } + } + } + }, + "400": { + "description": "Bad request" + }, + "403": { + "description": "Owner role required" + }, + "502": { + "description": "Repository fetch failed" + }, + "503": { + "description": "No forge client or no workflow store" + } + } + } } }, "components": { @@ -39759,6 +39832,38 @@ "type": "object", "description": "Compiled, derivation-ready form of a model (hash, states, columns, ordered rules). Never trusted from the wire: the spoke recompiles on save and on bind.", "additionalProperties": true + }, + "WorkflowIngestResult": { + "type": "object", + "properties": { + "model": { + "$ref": "#/components/schemas/WorkflowModel" + }, + "compiled": { + "type": "boolean" + }, + "diagnostics": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WorkflowDiagnostic" + } + }, + "profile": { + "type": "object" + }, + "ref": { + "type": "string" + }, + "files": { + "type": "integer" + }, + "pipelines": { + "type": "integer" + }, + "entry": { + "$ref": "#/components/schemas/WorkflowEntry" + } + } } } } diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 631231f642..1fdb6a04d7 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1087,6 +1087,7 @@ func (b *boot) wireBootClosures() { return &dashboard.Dependencies{ Config: b.cfg, WorkflowStore: wfstore.Open(wfstore.Root(b.cfg.Data.WorkflowsDir)), + WorkflowRepoFiles: workflowRepoFiles(func() *github.Client { return b.ghClient }), AgentMgr: b.agentMgr, CruxRuntimeHealth: cruxRuntimeHealthFunc(b.cruxSupervision), // Provider gateways (#5565 slice 3): concrete openrouter/watsonx/ diff --git a/src/cmd/hive/workflow_ingest.go b/src/cmd/hive/workflow_ingest.go new file mode 100644 index 0000000000..00f80f6d45 --- /dev/null +++ b/src/cmd/hive/workflow_ingest.go @@ -0,0 +1,37 @@ +package main + +import ( + "context" + "errors" + + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest" +) + +// errNoForgeClient is returned by the workflow ingest fetcher before a +// GitHub client exists. +var errNoForgeClient = errors.New("no GitHub client") + +// workflowRepoFiles is the dashboard's WorkflowRepoFiles dependency +// (POST /api/workflows/ingest, RFC #17620 P3): it downloads the repo's +// default branch read-only, keeps only the CI definition files and +// removes the checkout. client is read per call so a credential rebuild is +// followed. +func workflowRepoFiles(client func() *github.Client) func(ctx context.Context, repo string) (map[string][]byte, string, error) { + return func(ctx context.Context, repo string) (map[string][]byte, string, error) { + c := client() + if c == nil { + return nil, "", errNoForgeClient + } + co, err := c.FetchDefaultBranchTree(ctx, repo) + if err != nil { + return nil, "", err + } + defer co.Cleanup() + files, err := ingest.ReadTree(co.Dir) + if err != nil { + return nil, "", err + } + return files, co.Ref, nil + } +} diff --git a/src/cmd/hive/workflow_ingest_test.go b/src/cmd/hive/workflow_ingest_test.go new file mode 100644 index 0000000000..b10e3e219c --- /dev/null +++ b/src/cmd/hive/workflow_ingest_test.go @@ -0,0 +1,16 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/hivecommons/hive/pkg/github" +) + +func TestWorkflowRepoFilesWithoutClient(t *testing.T) { + fetch := workflowRepoFiles(func() *github.Client { return nil }) + if _, _, err := fetch(context.Background(), "acme/widgets"); !errors.Is(err, errNoForgeClient) { + t.Fatalf("err = %v", err) + } +} diff --git a/src/cmd/workflow-survey/main.go b/src/cmd/workflow-survey/main.go index 45f352b3b9..31ad3e2752 100644 --- a/src/cmd/workflow-survey/main.go +++ b/src/cmd/workflow-survey/main.go @@ -68,6 +68,12 @@ type repoResult struct { ValidateError string `json:"validateError,omitempty"` Hash string `json:"hash"` Severities map[workflowmodel.Severity]int `json:"severities,omitempty"` + // Compiled says whether the lifecycle model (ingest.Lifecycle) compiles; + // CompileCodes counts its diagnostics by code, errors and warnings. + Compiled bool `json:"compiled"` + CompileError string `json:"compileError,omitempty"` + CompileCodes map[string]int `json:"compileCodes,omitempty"` + LifecycleCols int `json:"lifecycleColumns"` } type summary struct { @@ -78,6 +84,8 @@ type summary struct { Parsed int `json:"parsed"` Pipelines int `json:"pipelines"` Valid int `json:"valid"` + Compiled int `json:"compiled"` + CompileCodes map[string]int `json:"compileCodes"` Roles map[workflowmodel.Role]int `json:"roles"` Constructs map[string]int `json:"constructs"` FindingCodes map[string]int `json:"findingCodes"` @@ -160,7 +168,7 @@ func run(corpus, out string, filter map[string]bool, stdout, stderr io.Writer) e return err } maturity := loadProjects(corpus) - sum := &summary{Roles: map[workflowmodel.Role]int{}, Constructs: map[string]int{}, FindingCodes: map[string]int{}, Severities: map[workflowmodel.Severity]int{}, ColumnSets: map[string]int{}, ByProject: map[string]*projectStats{}} + sum := &summary{Roles: map[workflowmodel.Role]int{}, Constructs: map[string]int{}, FindingCodes: map[string]int{}, Severities: map[workflowmodel.Severity]int{}, CompileCodes: map[string]int{}, ColumnSets: map[string]int{}, ByProject: map[string]*projectStats{}} for _, d := range dirs { if !d.IsDir() { continue @@ -252,9 +260,25 @@ func surveyRepo(dir, repo, outDir string) (repoResult, error) { } res.ColumnDiff = render.Diff(m) + lm := ingest.Lifecycle(workflowmodel.Scope{Forge: defaultForge, Repository: repo}, files) + res.LifecycleCols = len(lm.Spec.Columns) + res.CompileCodes = map[string]int{} + _, diags, cerr := workflowmodel.Compile(lm) + for _, d := range diags { + res.CompileCodes[d.Code]++ + } + if cerr != nil { + res.CompileError = cerr.Error() + } else { + res.Compiled = true + } + if err := os.MkdirAll(outDir, 0o755); err != nil { return res, err } + if err := writeJSON(filepath.Join(outDir, "lifecycle.json"), lm); err != nil { + return res, err + } if err := writeJSON(filepath.Join(outDir, "model.json"), m); err != nil { return res, err } @@ -324,6 +348,12 @@ func accumulate(s *summary, r repoResult) { if r.Valid { s.Valid++ } + if r.Compiled { + s.Compiled++ + } + for k, v := range r.CompileCodes { + s.CompileCodes[k] += v + } if len(r.ColumnDiff) > 0 { s.ReposChangedCols++ } @@ -375,7 +405,12 @@ func markdown(s *summary) string { fmt.Fprintf(&b, "# Workflow model survey\n\n") fmt.Fprintf(&b, "| | |\n|---|---|\n") fmt.Fprintf(&b, "| Repositories | %d |\n| with GitHub Actions pipelines | %d |\n| with Tekton pipelines | %d |\n", s.Repos, s.ReposWithGHA, s.ReposWithTekton) - fmt.Fprintf(&b, "| Files read | %d |\n| Files parsed | %d (%.1f%%) |\n| Pipelines | %d |\n| Models valid | %d / %d |\n| Repos whose columns change | %d |\n\n", s.Files, s.Parsed, pct(s.Parsed, s.Files), s.Pipelines, s.Valid, s.Repos, s.ReposChangedCols) + fmt.Fprintf(&b, "| Files read | %d |\n| Files parsed | %d (%.1f%%) |\n| Pipelines | %d |\n| Models valid | %d / %d |\n| Lifecycle models compiled | %d / %d |\n| Repos whose columns change | %d |\n\n", s.Files, s.Parsed, pct(s.Parsed, s.Files), s.Pipelines, s.Valid, s.Repos, s.Compiled, s.Repos, s.ReposChangedCols) + b.WriteString("## Lifecycle compile diagnostics\n\n| Code | Count |\n|---|---|\n") + for _, k := range sortedKeys(s.CompileCodes) { + fmt.Fprintf(&b, "| %s | %d |\n", k, s.CompileCodes[k]) + } + b.WriteString("\n") b.WriteString("## Pipeline roles\n\n| Role | Pipelines |\n|---|---|\n") for _, k := range sortedKeys(s.Roles) { fmt.Fprintf(&b, "| %s | %d |\n", k, s.Roles[workflowmodel.Role(k)]) diff --git a/src/docs/design/workflow-models.md b/src/docs/design/workflow-models.md index 13c080eeb2..c98e896da6 100644 --- a/src/docs/design/workflow-models.md +++ b/src/docs/design/workflow-models.md @@ -125,3 +125,33 @@ kind is not covered. Models are whole (no inheritance) through phase 4. columns, numbers, error}}}` (nil when nothing is bound). The JS labels the model column, renders a per-row column strip by stage number, and shows an amber "Not covered" chevron for abstentions. Settings → Repos has a per-repo Workflow select that PUTs the binding. + +## Ingest → lifecycle (RFC phase 3a) + +`ingest.Repository` (phase 0) is unchanged and still produces the process half. `ingest.Lifecycle` +composes on top of it: + +1. Start from `hivedefault.Generate()` — the compiled hive-default lifecycle, `derivation: builtin`. +2. Copy the ingested pipelines, sources and coverage; `meta.source` is the system(s) read, + `meta.provenance` is `ingested-unreviewed`, `meta.version` is `1` (the store assigns later + versions). +3. PR-verify pipelines are bound to every state in the CI / Review column, each with a citation to + the file the pipeline came from. +4. Every post-merge state the process half produced (main CI, release, `deploy-`, done) + becomes a state in its own column after Merged, `confidence: inferred`, with the environment + guards carried over. The chain's transitions fire on `pipeline-succeeded` = + `pipeline.status is green`. The engine does not observe post-merge runs yet and reports + `unknown` for that fact, so a bound ingested model derives exactly like hive-default today; + the extra columns render, nothing lands in them until post-merge observation exists. + +The compiler warns once per inferred element; a human confirming them in the editor (phase 4) +is what moves provenance to `local`/`curated`. + +`POST /api/workflows/ingest` is the spoke entry point (owner role): `{repo}` fetches the default +branch through `FetchDefaultBranchTree`, `ingest.ReadTree` keeps GitHub Actions workflows, YAML +under Tekton directories and pipeline-named YAML that looks like Tekton (the survey collector's +rules), and the checkout is removed. `{files}` accepts pasted definitions. `save: true` stores the +result at the next integer version under `name` (default: the repository slug). + +`cmd/workflow-survey` now compiles every corpus repository's lifecycle model (`lifecycle.json` +alongside `model.json`) and reports pass/refusal counts by diagnostic code. diff --git a/src/pkg/dashboard/api_workflows.go b/src/pkg/dashboard/api_workflows.go index 025d1241f3..c1065c0952 100644 --- a/src/pkg/dashboard/api_workflows.go +++ b/src/pkg/dashboard/api_workflows.go @@ -20,6 +20,7 @@ func (s *Server) registerWorkflowRoutes() { s.mux.HandleFunc("GET /api/workflows", s.handleWorkflowList) s.mux.HandleFunc("POST /api/workflows", s.handleWorkflowSave) s.mux.HandleFunc("POST /api/workflows/compile", s.handleWorkflowCompile) + s.mux.HandleFunc("POST /api/workflows/ingest", s.handleWorkflowIngest) s.mux.HandleFunc("GET /api/workflows/bindings", s.handleWorkflowBindingsGet) s.mux.HandleFunc("PUT /api/workflows/bindings", s.handleWorkflowBindingPut) s.mux.HandleFunc("GET /api/workflows/{name}/{version}", s.handleWorkflowGet) diff --git a/src/pkg/dashboard/api_workflows_ingest.go b/src/pkg/dashboard/api_workflows_ingest.go new file mode 100644 index 0000000000..155f08a24f --- /dev/null +++ b/src/pkg/dashboard/api_workflows_ingest.go @@ -0,0 +1,134 @@ +package dashboard + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest" + wfstore "github.com/hivecommons/hive/pkg/workflowmodel/store" +) + +// workflowIngestTimeout bounds one repository fetch + ingest. +const workflowIngestTimeout = 2 * time.Minute + +// workflowIngestRequest is the body of POST /api/workflows/ingest. Exactly +// one of Repo (fetch the default branch through the forge) or Files +// (pasted/uploaded CI definitions, repo-relative path -> contents) is set. +type workflowIngestRequest struct { + Repo string `json:"repo"` + Files map[string]string `json:"files,omitempty"` + // Save stores the result as a draft under Name (default: the ingest's + // slug of the repository) at the next free integer version. + Save bool `json:"save,omitempty"` + Name string `json:"name,omitempty"` +} + +// handleWorkflowIngest turns a repository's CI definitions into a lifecycle +// model (RFC #17620 P3): hive-default's stages plus the repository's +// pipelines, provenance ingested-unreviewed. Owner-gated because it reads +// the forge on the hive's credential and may write the store. +func (s *Server) handleWorkflowIngest(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + var req workflowIngestRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + req.Repo = strings.TrimSpace(req.Repo) + if (req.Repo == "") == (len(req.Files) == 0) { + jsonError(w, "set exactly one of repo or files", http.StatusBadRequest) + return + } + var files map[string][]byte + ref := "" + repo := req.Repo + switch { + case len(req.Files) > 0: + if repo == "" { + repo = "local/pasted" + } + files = make(map[string][]byte, len(req.Files)) + for p, body := range req.Files { + files[strings.TrimPrefix(strings.TrimSpace(p), "/")] = []byte(body) + } + default: + if !strings.Contains(repo, "/") { + jsonError(w, "repo must be owner/name", http.StatusBadRequest) + return + } + if s.deps == nil || s.deps.WorkflowRepoFiles == nil { + jsonError(w, "repository ingest not available: no forge client", http.StatusServiceUnavailable) + return + } + ctx, cancel := context.WithTimeout(r.Context(), workflowIngestTimeout) + defer cancel() + var err error + files, ref, err = s.deps.WorkflowRepoFiles(ctx, repo) + if err != nil { + jsonError(w, fmt.Sprintf("fetch %s: %v", repo, err), http.StatusBadGateway) + return + } + } + + m := ingest.Lifecycle(workflowmodel.Scope{Forge: "github.com", Repository: repo}, files) + if req.Name != "" { + m.Metadata.Name = req.Name + } + p, diags, cerr := workflowmodel.Compile(m) + if diags == nil { + diags = []workflowmodel.Diagnostic{} + } + resp := map[string]interface{}{ + "model": m, "compiled": cerr == nil, "diagnostics": diags, + "ref": ref, "files": len(files), "pipelines": len(m.Spec.Pipelines), + } + if p != nil { + resp["profile"] = p + } + if !req.Save { + jsonResponse(w, resp) + return + } + st := s.workflowStore(w) + if st == nil { + return + } + version, err := nextWorkflowVersion(st, m.Metadata.Name) + if err != nil { + workflowStoreError(w, err) + return + } + m.Spec.Meta.Version = version + e, err := st.Save(m, requestUser(r), time.Now().UTC()) + if err != nil { + workflowStoreError(w, err) + return + } + s.auditFromRequest(r, "workflow_ingest", fmt.Sprintf("%s from %s@%s files=%d compiled=%v", e.Key(), repo, ref, len(files), e.Compiled), "") + resp["entry"] = e + jsonStatusResponse(w, http.StatusCreated, resp) +} + +// nextWorkflowVersion is one past the highest integer version stored under +// name ("1" for a new name); non-integer versions are ignored. +func nextWorkflowVersion(st *wfstore.Store, name string) (string, error) { + entries, err := st.Versions(name) + if err != nil && !errors.Is(err, wfstore.ErrNotFound) { + return "", err + } + max := 0 + for _, e := range entries { + if n, err := strconv.Atoi(e.Version); err == nil && n > max { + max = n + } + } + return strconv.Itoa(max + 1), nil +} diff --git a/src/pkg/dashboard/api_workflows_ingest_test.go b/src/pkg/dashboard/api_workflows_ingest_test.go new file mode 100644 index 0000000000..ec96d72e6a --- /dev/null +++ b/src/pkg/dashboard/api_workflows_ingest_test.go @@ -0,0 +1,105 @@ +package dashboard + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "testing" + + "github.com/hivecommons/hive/pkg/workflowmodel" + wfstore "github.com/hivecommons/hive/pkg/workflowmodel/store" +) + +const ingestCIYAML = "name: CI\non: [pull_request, push]\njobs:\n test:\n runs-on: ubuntu-latest\n steps: [{run: make test}]\n" + +func TestWorkflowIngestFromFilesCompilesAndSaves(t *testing.T) { + s := workflowTestServer(t, true) + body := map[string]any{"files": map[string]string{".github/workflows/ci.yml": ingestCIYAML}, "name": "pasted", "save": true} + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest", body, true) + if rec.Code != http.StatusCreated { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + var resp struct { + Compiled bool `json:"compiled"` + Pipelines int `json:"pipelines"` + Entry wfstore.Entry `json:"entry"` + Model workflowmodel.Model `json:"model"` + Diags []workflowmodel.Diagnostic `json:"diagnostics"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if !resp.Compiled || resp.Pipelines != 1 || resp.Entry.Key() != "pasted@1" || !resp.Entry.Compiled { + t.Fatalf("%+v", resp) + } + if resp.Model.Spec.Meta.Provenance != workflowmodel.ProvenanceIngestedUnreviewed { + t.Fatalf("provenance %q", resp.Model.Spec.Meta.Provenance) + } + // A second ingest of the same name lands at the next version. + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest", body, true) + if rec.Code != http.StatusCreated { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.Entry.Key() != "pasted@2" { + t.Fatalf("second version %s", resp.Entry.Key()) + } + // Dry run (no save) needs no store and returns the model. + s2 := workflowTestServer(t, false) + rec = wfReq(s2, http.MethodPost, "/api/workflows/ingest", map[string]any{"files": map[string]string{".github/workflows/ci.yml": ingestCIYAML}}, true) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body) + } +} + +func TestWorkflowIngestFromRepoUsesForgeFetcher(t *testing.T) { + s := workflowTestServer(t, true) + var asked string + s.deps.WorkflowRepoFiles = func(_ context.Context, repo string) (map[string][]byte, string, error) { + asked = repo + if repo == "acme/missing" { + return nil, "", errors.New("404") + } + return map[string][]byte{".github/workflows/ci.yml": []byte(ingestCIYAML)}, "main", nil + } + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest", map[string]any{"repo": "acme/widgets"}, true) + if rec.Code != http.StatusOK || asked != "acme/widgets" { + t.Fatalf("%d %s asked=%q", rec.Code, rec.Body, asked) + } + var resp struct { + Ref string `json:"ref"` + Model workflowmodel.Model `json:"model"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.Ref != "main" || resp.Model.Metadata.Name != "acme-widgets" || resp.Model.Spec.Scope.Repository != "acme/widgets" { + t.Fatalf("%+v", resp) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest", map[string]any{"repo": "acme/missing"}, true); rec.Code != http.StatusBadGateway { + t.Fatalf("fetch failure %d", rec.Code) + } +} + +func TestWorkflowIngestGuards(t *testing.T) { + s := workflowTestServer(t, true) + cases := []struct { + body map[string]any + owner bool + want int + }{ + {map[string]any{"repo": "acme/widgets"}, false, http.StatusForbidden}, + {map[string]any{}, true, http.StatusBadRequest}, + {map[string]any{"repo": "acme/widgets", "files": map[string]string{"a.yml": "x"}}, true, http.StatusBadRequest}, + {map[string]any{"repo": "nojslash"}, true, http.StatusBadRequest}, + {map[string]any{"repo": "acme/widgets"}, true, http.StatusServiceUnavailable}, // no fetcher + } + for i, c := range cases { + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest", c.body, c.owner); rec.Code != c.want { + t.Fatalf("case %d: %d %s", i, rec.Code, rec.Body) + } + } +} diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index a43aa6f9dc..b796186b11 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -61,6 +61,10 @@ type Dependencies struct { // WorkflowStore serves the workflow-model library (hivecommons/hive#17620). // Nil disables the /api/workflows endpoints. WorkflowStore *wfstore.Store + // WorkflowRepoFiles fetches a repository's default-branch CI definition + // files (repo-relative slash path -> contents) and the ref read, for + // POST /api/workflows/ingest. Nil disables ingest from a repository. + WorkflowRepoFiles func(ctx context.Context, repo string) (files map[string][]byte, ref string, err error) // AgentMgr is the agent runtime (hivecommons/hive#13536). The dashboard // programs against the crux.Runtime surface, not *agent.Manager; tests that // need manager-only seams unwrap it with agentManager. diff --git a/src/pkg/stageengine/workflow.go b/src/pkg/stageengine/workflow.go index c01674898d..f344eef66f 100644 --- a/src/pkg/stageengine/workflow.go +++ b/src/pkg/stageengine/workflow.go @@ -276,6 +276,9 @@ func scalarFact(f Facts, name string) (string, bool) { return "", true } return f.PR.BaseRef, true + case "pipeline.status": + // Post-merge runs are not observed yet (see workflowmodel.Facts). + return "unknown", true } return "", false } diff --git a/src/pkg/stageengine/workflow_rules_test.go b/src/pkg/stageengine/workflow_rules_test.go index 2af5cfe357..418a661983 100644 --- a/src/pkg/stageengine/workflow_rules_test.go +++ b/src/pkg/stageengine/workflow_rules_test.go @@ -201,6 +201,9 @@ func scalarValue(fact string) string { return "changes_requested" case "pr.dco": return "yes" + case "pipeline.status": + // Post-merge runs are not observed yet; the engine reports unknown. + return "unknown" case "pr.base": return "main" } diff --git a/src/pkg/workflowmodel/ingest/lifecycle.go b/src/pkg/workflowmodel/ingest/lifecycle.go new file mode 100644 index 0000000000..b231f66f18 --- /dev/null +++ b/src/pkg/workflowmodel/ingest/lifecycle.go @@ -0,0 +1,139 @@ +package ingest + +import ( + "fmt" + "sort" + "strings" + + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/hivedefault" +) + +// LifecycleVersion is the spec.meta.version every ingested lifecycle +// model starts at; the store and marketplace assign later versions. +const LifecycleVersion = "1" + +// EventPipelineSucceeded moves work along the ingested post-merge chain. +const EventPipelineSucceeded = "pipeline-succeeded" + +// Lifecycle ingests files for scope and returns a model with the RFC's +// lifecycle half filled in so that it compiles: the hive-default lifecycle +// (states, columns, builtin derivation, roles, gates, events) plus the +// repository's pipelines bound where they run — PR-verify pipelines on the +// CI / Review states, and the post-merge chain (main CI, release, deploys, +// done) as new columns after Merged. Everything the ingest added is +// origin ingested, confidence inferred (the compiler warns per element) +// and the model's provenance is ingested-unreviewed until a human edits it. +// +// The process half (Repository) is unchanged; this composes on top of it. +func Lifecycle(scope workflowmodel.Scope, files map[string][]byte) *workflowmodel.Model { + base := Repository(scope, files) + m := hivedefault.Generate() + m.Metadata.Name = base.Metadata.Name + m.Metadata.Annotations = base.Metadata.Annotations + m.Spec.Scope = scope + m.Spec.Meta.Version = LifecycleVersion + m.Spec.Meta.Author = "" + m.Spec.Meta.Description = fmt.Sprintf("Lifecycle ingested from %s: hive-default stages with the repository's pipelines bound", scope.Repository) + m.Spec.Meta.Source = ingestSource(base) + m.Spec.Meta.Provenance = workflowmodel.ProvenanceIngestedUnreviewed + m.Spec.Pipelines = base.Spec.Pipelines + m.Spec.Sources = base.Spec.Sources + m.Spec.Coverage = base.Spec.Coverage + if len(base.Spec.Pipelines) == 0 { + return m + } + + cite := citations(base.Spec.Pipelines) + + // PR-verify pipelines run on every head in the CI / Review column. + if ci := base.FindState(workflowmodel.StateCI); ci != nil && len(ci.Pipelines) > 0 { + for i := range m.Spec.States { + s := &m.Spec.States[i] + if s.Lifecycle.Column == hivedefault.ColumnCIReview { + s.Pipelines = append([]string(nil), ci.Pipelines...) + s.Lifecycle.Citations = cite(ci.Pipelines) + } + } + } + + // The post-merge chain: every ingested state after Merged, in order. + var post []workflowmodel.State + for _, s := range base.Spec.States { + if s.Origin == workflowmodel.OriginIngested && s.ID != workflowmodel.StateCI { + post = append(post, s) + } + } + if len(post) == 0 { + return m + } + for _, g := range base.Spec.Guards { + if g.Kind == workflowmodel.GuardEnvironmentApproval { + m.Spec.Guards = append(m.Spec.Guards, g) + } + } + m.Spec.Events = append(m.Spec.Events, workflowmodel.Event{ + ID: EventPipelineSucceeded, Label: "Post-merge pipeline succeeded", + Fact: workflowmodel.Predicate{Fact: "pipeline.status", Op: workflowmodel.OpIs, Value: "green"}, + Confidence: workflowmodel.ConfidenceInferred, + }) + prev := hivedefault.StateMerged + for _, s := range post { + st := s + st.Lifecycle = workflowmodel.StateLifecycle{Column: st.ID, Kinds: []workflowmodel.ItemKind{workflowmodel.KindPR}, Confidence: workflowmodel.ConfidenceInferred, Citations: cite(st.Pipelines)} + col := workflowmodel.Column{ID: st.ID, Label: st.Label, States: []string{st.ID}, Terminal: st.Type == workflowmodel.StateFinal} + if st.ID == workflowmodel.StateDone { + col.Hint = "Every post-merge pipeline has run" + } else { + col.Hint = "Runs after merge: " + strings.Join(st.Pipelines, ", ") + } + m.Spec.States = append(m.Spec.States, st) + m.Spec.Columns = append(m.Spec.Columns, col) + m.Spec.Transitions = append(m.Spec.Transitions, workflowmodel.Transition{From: prev, To: st.ID, On: EventPipelineSucceeded, GuardRef: st.GuardRef, Note: "ingested post-merge chain"}) + prev = st.ID + } + return m +} + +// ingestSource is the meta source for the systems the pipelines came from. +func ingestSource(m *workflowmodel.Model) workflowmodel.MetaSource { + var gha, tkn bool + for _, p := range m.Spec.Pipelines { + if p.System == workflowmodel.SystemTekton { + tkn = true + } else { + gha = true + } + } + switch { + case gha && !tkn: + return workflowmodel.SourceGitHubActions + case tkn && !gha: + return workflowmodel.SourceTekton + case gha && tkn: + return workflowmodel.SourceGitHubActions + "+" + workflowmodel.SourceTekton + } + return "" +} + +// citations returns a lookup from pipeline ids to the files they came from. +func citations(pipelines []workflowmodel.Pipeline) func(ids []string) []workflowmodel.Citation { + paths := map[string]string{} + for _, p := range pipelines { + paths[p.ID] = p.Path + } + return func(ids []string) []workflowmodel.Citation { + seen := map[string]bool{} + var out []workflowmodel.Citation + for _, id := range ids { + p, ok := paths[id] + if !ok || seen[p] { + continue + } + seen[p] = true + out = append(out, workflowmodel.Citation{Path: p, Text: id}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out + } +} diff --git a/src/pkg/workflowmodel/ingest/lifecycle_test.go b/src/pkg/workflowmodel/ingest/lifecycle_test.go new file mode 100644 index 0000000000..f37f884001 --- /dev/null +++ b/src/pkg/workflowmodel/ingest/lifecycle_test.go @@ -0,0 +1,144 @@ +package ingest + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/hivedefault" +) + +// Every corpus repository must yield a lifecycle model that compiles; +// refusals are the ingest's bug, warnings are the inferred elements. +func TestLifecycleCorpusCompiles(t *testing.T) { + entries, err := os.ReadDir(corpusDir) + if err != nil { + t.Fatal(err) + } + seen := 0 + for _, e := range entries { + if !e.IsDir() { + continue + } + seen++ + t.Run(e.Name(), func(t *testing.T) { + repo := strings.ReplaceAll(e.Name(), "__", "/") + m := Lifecycle(workflowmodel.Scope{Forge: "github.com", Repository: repo}, readRepo(t, filepath.Join(corpusDir, e.Name()))) + p, diags, err := workflowmodel.Compile(m) + if err != nil { + t.Fatalf("compile: %v\n%v", err, diags) + } + if p.Mode != workflowmodel.DerivationBuiltin { + t.Fatalf("mode %q", p.Mode) + } + if m.Spec.Meta.Provenance != workflowmodel.ProvenanceIngestedUnreviewed || m.Spec.Meta.Version == "" { + t.Fatalf("meta %+v", m.Spec.Meta) + } + if len(m.Spec.Pipelines) == 0 { + t.Fatal("corpus repo ingested no pipelines") + } + // The hive-default lifecycle is intact underneath. + for _, s := range hivedefault.Generate().Spec.States { + if _, ok := p.States[s.ID]; !ok { + t.Fatalf("hive-default state %q missing", s.ID) + } + } + // Every ingested element is inferred and warned about. + inferred := 0 + for _, d := range diags { + if d.Code == workflowmodel.CodeInferred { + inferred++ + } + } + ingested := 0 + for _, s := range m.Spec.States { + if s.Origin == workflowmodel.OriginIngested { + ingested++ + if s.Lifecycle.Confidence != workflowmodel.ConfidenceInferred || s.Lifecycle.Column != s.ID { + t.Fatalf("ingested state %+v", s.Lifecycle) + } + if s.ID != workflowmodel.StateDone && len(s.Lifecycle.Citations) == 0 { + t.Fatalf("ingested state %q has no citations", s.ID) + } + } + } + if ingested > 0 && inferred == 0 { + t.Fatalf("%d ingested states but no inferred warnings", ingested) + } + }) + } + if seen == 0 { + t.Fatal("empty corpus") + } +} + +func TestLifecyclePostMergeChain(t *testing.T) { + files := map[string][]byte{ + ".github/workflows/ci.yml": []byte("name: CI\non: [pull_request, push]\njobs:\n test:\n runs-on: ubuntu-latest\n steps: [{run: make test}]\n"), + ".github/workflows/release.yml": []byte("name: Release\non:\n push:\n tags: ['v*']\njobs:\n publish:\n runs-on: ubuntu-latest\n environment: production\n steps: [{run: make publish}]\n"), + } + m := Lifecycle(workflowmodel.Scope{Forge: "github.com", Repository: "acme/widgets"}, files) + p, diags, err := workflowmodel.Compile(m) + if err != nil { + t.Fatalf("%v %v", err, diags) + } + if m.Spec.Meta.Source != workflowmodel.SourceGitHubActions { + t.Fatalf("source %q", m.Spec.Meta.Source) + } + // Columns: hive-default's seven, then main CI, release, done. + var cols []string + for _, c := range m.Spec.Columns { + cols = append(cols, c.ID) + } + got := strings.Join(cols, " ") + if !strings.HasSuffix(got, hivedefault.ColumnMerged+" "+workflowmodel.StateMainCI+" "+workflowmodel.StateRelease+" "+workflowmodel.StateDone) { + t.Fatalf("columns %s", got) + } + if !p.Terminal[workflowmodel.StateDone] || !p.Terminal[hivedefault.StateMerged] { + t.Fatalf("terminal %v", p.Terminal) + } + // PR-verify pipelines sit on every CI / Review state with a citation. + bound := 0 + for _, s := range m.Spec.States { + if s.Lifecycle.Column == hivedefault.ColumnCIReview { + if len(s.Pipelines) != 1 || len(s.Lifecycle.Citations) != 1 || s.Lifecycle.Citations[0].Path != ".github/workflows/ci.yml" { + t.Fatalf("ci state %q: %v %v", s.ID, s.Pipelines, s.Lifecycle.Citations) + } + bound++ + } + } + if bound == 0 { + t.Fatal("no CI / Review state bound") + } + // The chain fires on the declared pipeline event, merged -> main-ci -> release -> done. + var chain []string + for _, tr := range m.Spec.Transitions { + if tr.On == EventPipelineSucceeded { + chain = append(chain, tr.From+">"+tr.To) + } + } + if want := "merged>main-ci main-ci>release release>done"; strings.Join(chain, " ") != want { + t.Fatalf("chain %v", chain) + } +} + +func TestLifecycleNoPipelinesIsHiveDefaultLifecycle(t *testing.T) { + m := Lifecycle(workflowmodel.Scope{Forge: "github.com", Repository: "acme/empty"}, map[string][]byte{"README.md": []byte("hi")}) + p, _, err := workflowmodel.Compile(m) + if err != nil { + t.Fatal(err) + } + ref, _, _ := workflowmodel.Compile(hivedefault.Generate()) + if len(p.States) != len(ref.States) || len(m.Spec.Columns) != len(hivedefault.Generate().Spec.Columns) { + t.Fatalf("states %d/%d", len(p.States), len(ref.States)) + } + if m.Spec.Meta.Source != "" || m.Spec.Meta.Provenance != workflowmodel.ProvenanceIngestedUnreviewed { + t.Fatalf("meta %+v", m.Spec.Meta) + } +} + +func scopeFor(repo string) workflowmodel.Scope { + return workflowmodel.Scope{Forge: "github.com", Repository: repo} +} diff --git a/src/pkg/workflowmodel/ingest/tree.go b/src/pkg/workflowmodel/ingest/tree.go new file mode 100644 index 0000000000..570674ef2f --- /dev/null +++ b/src/pkg/workflowmodel/ingest/tree.go @@ -0,0 +1,90 @@ +package ingest + +import ( + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/hivecommons/hive/pkg/workflowmodel/githubactions" + "github.com/hivecommons/hive/pkg/workflowmodel/tekton" +) + +// MaxTreeFiles bounds how many CI files ReadTree returns from one +// checkout; a monorepo past this is reported, not read to exhaustion. +const MaxTreeFiles = 2000 + +// tektonDirs are the directories Tekton resources conventionally live in +// (the same list the survey collector used, so a spoke ingest reads what +// the corpus read). +var tektonDirs = []string{".tekton/", "tekton/", "pipelines/", ".pipelines/", "openshift/ci/", "ci/tekton/"} + +// skipDirs are never descended into. +var skipDirs = map[string]bool{".git": true, "node_modules": true, "vendor": true, "testdata": true, "third_party": true} + +// ReadTree walks a checkout and returns the CI definition files the ingest +// reads, keyed by repo-relative slash path: every GitHub Actions workflow, +// every YAML under a Tekton directory, and any other YAML whose name hints +// at a pipeline and whose content looks like a Tekton document. Files over +// MaxFileBytes are returned too (Repository reports them). The result is +// what Repository and Lifecycle take. +func ReadTree(root string) (map[string][]byte, error) { + files := map[string][]byte{} + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, rerr := filepath.Rel(root, p) + if rerr != nil { + return rerr + } + rel = filepath.ToSlash(rel) + if d.IsDir() { + if rel != "." && skipDirs[d.Name()] { + return filepath.SkipDir + } + return nil + } + if !d.Type().IsRegular() || !isYAMLPath(rel) || len(files) >= MaxTreeFiles { + return nil + } + switch { + case githubactions.IsWorkflowPath(rel), underTektonDir(rel): + case hintsPipeline(rel): + data, err := os.ReadFile(p) + if err != nil { + return err + } + if tekton.LooksTekton(data) { + files[rel] = data + } + return nil + default: + return nil + } + data, err := os.ReadFile(p) + if err != nil { + return err + } + files[rel] = data + return nil + }) + if err != nil { + return nil, err + } + return files, nil +} + +func underTektonDir(rel string) bool { + for _, d := range tektonDirs { + if strings.HasPrefix(rel, d) { + return true + } + } + return false +} + +func hintsPipeline(rel string) bool { + base := strings.ToLower(filepath.Base(rel)) + return strings.Contains(base, "pipeline") || strings.Contains(base, "tekton") +} diff --git a/src/pkg/workflowmodel/ingest/tree_test.go b/src/pkg/workflowmodel/ingest/tree_test.go new file mode 100644 index 0000000000..6c25e1b936 --- /dev/null +++ b/src/pkg/workflowmodel/ingest/tree_test.go @@ -0,0 +1,66 @@ +package ingest + +import ( + "os" + "path/filepath" + "sort" + "strings" + "testing" +) + +func TestReadTreeSelectsCIFiles(t *testing.T) { + root := t.TempDir() + write := func(rel, body string) { + t.Helper() + p := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write(".github/workflows/ci.yml", "on: push\njobs: {}\n") + write(".github/workflows/nested/no.yml", "on: push\n") + write(".github/dependabot.yml", "version: 2\n") + write(".tekton/pr.yaml", "apiVersion: tekton.dev/v1\nkind: PipelineRun\n") + write("ci/tekton/build.yaml", "kind: Task\n") + write("deploy/release-pipeline.yaml", "apiVersion: tekton.dev/v1\nkind: Pipeline\n") + write("deploy/other-pipeline.yaml", "apiVersion: apps/v1\nkind: Deployment\n") + write("charts/values.yaml", "replicas: 1\n") + write("vendor/x/.github/workflows/v.yml", "on: push\n") + write("testdata/pipeline.yaml", "apiVersion: tekton.dev/v1\n") + write("README.md", "hi\n") + + files, err := ReadTree(root) + if err != nil { + t.Fatal(err) + } + var got []string + for p := range files { + got = append(got, p) + } + sort.Strings(got) + want := ".github/workflows/ci.yml .tekton/pr.yaml ci/tekton/build.yaml deploy/release-pipeline.yaml" + if strings.Join(got, " ") != want { + t.Fatalf("got %v", got) + } +} + +func TestReadTreeFeedsLifecycle(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, ".github", "workflows"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, ".github", "workflows", "ci.yml"), []byte("name: CI\non: pull_request\njobs:\n t:\n runs-on: ubuntu-latest\n steps: [{run: true}]\n"), 0o644); err != nil { + t.Fatal(err) + } + files, err := ReadTree(root) + if err != nil { + t.Fatal(err) + } + m := Lifecycle(scopeFor("acme/x"), files) + if len(m.Spec.Pipelines) != 1 { + t.Fatalf("pipelines %d", len(m.Spec.Pipelines)) + } +} diff --git a/src/pkg/workflowmodel/lifecycle.go b/src/pkg/workflowmodel/lifecycle.go index 8fd8949bae..5860bdac49 100644 --- a/src/pkg/workflowmodel/lifecycle.go +++ b/src/pkg/workflowmodel/lifecycle.go @@ -183,6 +183,12 @@ var Facts = map[string]FactType{ "pr.dco": FactScalar, "pr.head_fresh": FactBool, "pr.base": FactScalar, + // pipeline.status is the latest post-merge run of the pipelines bound + // to the item's state: green | red | running | unknown. The engine does + // not observe post-merge runs yet and always reports unknown, so a + // transition on it never fires; it exists so ingested post-merge chains + // compile honestly instead of inventing a fact. + "pipeline.status": FactScalar, } // Event is something that moves work: a fact the engine observes, named