From 4137626cf3f2a3ecdf1afed06c652c7f0ae45776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sat, 14 Nov 2026 21:42:31 +0100 Subject: [PATCH 1/3] feat(cabinet): report whether the ProofTrace was persisted, and allow requiring it Setx logged a recording failure and returned a normal response with Warnings=nil, so a caller could not tell "execution succeeded" from "the proof was persisted". - SetResponse.Proof ({persisted, commit, error_code}) is returned on every successful execution; the recorder error text stays in the log, only a fixed error code reaches the response - ExecutionRecorder is unchanged; an optional CommitRecorder lets Setx return the persisted commit hash, and WorkflowRecorder implements it - WithEvidenceRequired / CIC_EVIDENCE_REQUIRED=true: an execution whose proof was not persisted (recording failed, or no recorder is configured) fails with ErrEvidenceRequired, mapped to HTTP 503 evidence_required; the error carries the chain hash because the workflow has already run - default stays non-fatal: HTTP 200 with proof.persisted=false and the trace kept - CIC_EVIDENCE_REQUIRED accepts only "true"/"false", so a typo cannot switch a required mode off SYMBOLS.md is regenerated for the new exported API. Refs #165 --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEUCIG5iHcypWDbQUGJr4Grc5pB7VgrQLtIBEoW99oGm2u9zAiEA879xHkJbkHOihs9iX59xjWxyIIht8YQXrYWp/76Qje0= hash-algorithm = sha256 digest = 7nm9tQyatYL523w/SYRZoB9m0rb0ETFbVSTtBbYse1o= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- cmd/relay/main.go | 30 +++- cmd/relay/main_api_test.go | 68 ++++++++ context/SYMBOLS.md | 6 + core/cabinet/api_set.go | 17 ++ core/cabinet/error.go | 8 + core/cabinet/execution_recorder.go | 12 +- core/cabinet/proof_status_test.go | 153 ++++++++++++++++++ core/cabinet/service.go | 57 ++++++- core/nexus/recorder/workflow_recorder.go | 12 +- .../recorder/workflow_recorder_commit_test.go | 67 ++++++++ 10 files changed, 416 insertions(+), 14 deletions(-) create mode 100644 core/cabinet/proof_status_test.go create mode 100644 core/nexus/recorder/workflow_recorder_commit_test.go diff --git a/cmd/relay/main.go b/cmd/relay/main.go index 519902de..2620383a 100644 --- a/cmd/relay/main.go +++ b/cmd/relay/main.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "strconv" + "strings" "time" "github.com/google/uuid" @@ -349,6 +350,13 @@ func (s *relayServer) setHandler(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusGatewayTimeout, "timeout", "Workflow execution timed out: "+err.Error()) return } + if cabinet.IsEvidenceRequired(err) { + // The workflow ran, but its proof was not persisted and evidence is required. + traceStatus = "error" + logEntry.WithField("outcome", "evidence_required").WithError(err).Error("Workflow executed but its proof was not persisted") + writeError(w, http.StatusServiceUnavailable, "evidence_required", err.Error()) + return + } if cabinet.IsInvalid(err) { traceStatus = "rejected" logEntry.WithField("outcome", "invalid_request").WithError(err).Warn("Invalid request") @@ -458,6 +466,19 @@ func loadSecretsFromVault(logger *logrus.Logger) *VaultSecrets { } } +// evidenceRequiredFromEnv reads CIC_EVIDENCE_REQUIRED. Unset or empty means false; only +// "true" and "false" are accepted, so a typo cannot silently switch a required mode off. +func evidenceRequiredFromEnv() (bool, error) { + switch v := strings.TrimSpace(os.Getenv("CIC_EVIDENCE_REQUIRED")); v { + case "", "false": + return false, nil + case "true": + return true, nil + default: + return false, fmt.Errorf("invalid CIC_EVIDENCE_REQUIRED %q (valid: \"true\", \"false\")", v) + } +} + // buildRecorderOption constructs a cabinet.WithRecorder option from the provided VaultSecrets. // If secrets is nil (Vault not configured), returns a no-op option. func buildRecorderOption(logger *logrus.Logger, secrets *VaultSecrets) cabinet.CabinetOption { @@ -625,6 +646,13 @@ func main() { } secrets := loadSecretsFromVault(logger) recorderOpt := buildRecorderOption(logger, secrets) + evidenceRequired, err := evidenceRequiredFromEnv() + if err != nil { + logger.WithError(err).Fatal("FATAL: invalid evidence configuration") + } + if evidenceRequired { + logger.Info("Evidence required: a workflow whose ProofTrace is not persisted fails with 503 evidence_required") + } // Build artifact signer for cic.artifact.sign — uses the same Vault key as the recorder. // Nil if Vault is not configured; cic_sign will be "unavailable" in that case (non-fatal). @@ -677,7 +705,7 @@ func main() { metricSink := obs.NewMultiMetricSink(logrusSink, promSink) gitOps.SetObsSinks(logrusSink, metricSink) // relay.git.op_duration_ms + WARN logok obsOpt := cabinet.WithObsSinks(logrusSink, metricSink) - cabinetSvc, err := cabinet.NewCabinetService(context.Background(), schemaStore, moduleStore, workflowStore, wasmConfig, recorderOpt, obsOpt) + cabinetSvc, err := cabinet.NewCabinetService(context.Background(), schemaStore, moduleStore, workflowStore, wasmConfig, recorderOpt, cabinet.WithEvidenceRequired(evidenceRequired), obsOpt) if err != nil { logger.WithField("error", err).Fatal("FATAL: Failed to create cabinet service") } diff --git a/cmd/relay/main_api_test.go b/cmd/relay/main_api_test.go index b907839f..c9e9581a 100644 --- a/cmd/relay/main_api_test.go +++ b/cmd/relay/main_api_test.go @@ -670,3 +670,71 @@ func TestSetHandler_SourceDigestFromTreeHash(t *testing.T) { server.setHandler(w, req) mockAPI.AssertExpectations(t) } + +// TestSetHandler_EvidenceRequired: the workflow ran, but its proof was not persisted +// and evidence is required — an explicit 503, never a plain 200. +func TestSetHandler_EvidenceRequired(t *testing.T) { + mockAPI := new(mockCabinetService) + mockAPI.On("GetWorkflow", mock.Anything).Return(&cabinet.Workflow{}, nil) + mockAPI.On("Setx", mock.Anything, mock.Anything).Return(nil, + fmt.Errorf("%w: workflow \"any\" executed but its proof was not persisted (record_failed), chain_hash=deadbeef", cabinet.ErrEvidenceRequired)) + + logger := logrus.New() + logger.SetOutput(new(bytes.Buffer)) + server := &relayServer{api: mockAPI, config: &config{MaxJSONInputSize: 1024}, logger: logger} + + req := httptest.NewRequest(http.MethodPost, "/set", bytes.NewReader([]byte(`{"workflow_id":"any"}`))) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + server.setHandler(w, req) + + assert.Equal(t, http.StatusServiceUnavailable, w.Code) + assertErrorJSON(t, w, "evidence_required") + assert.Contains(t, w.Body.String(), "chain_hash=deadbeef", "the chain hash must reach the caller") +} + +// TestSetHandler_ProofInResponse: a successful execution reports the proof outcome. +func TestSetHandler_ProofInResponse(t *testing.T) { + mockAPI := new(mockCabinetService) + mockAPI.On("GetWorkflow", mock.Anything).Return(&cabinet.Workflow{}, nil) + mockAPI.On("Setx", mock.Anything, mock.Anything).Return(&cabinet.SetResponse{ + Results: map[string]any{"ok": true}, + Proof: &cabinet.ProofStatus{Persisted: false, ErrorCode: cabinet.ProofErrRecordFailed}, + }, nil) + + logger := logrus.New() + logger.SetOutput(new(bytes.Buffer)) + server := &relayServer{api: mockAPI, config: &config{MaxJSONInputSize: 1024}, logger: logger} + + req := httptest.NewRequest(http.MethodPost, "/set", bytes.NewReader([]byte(`{"workflow_id":"any"}`))) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + + server.setHandler(w, req) + + assert.Equal(t, http.StatusOK, w.Code, "evidence not required: a successful execution keeps HTTP 200") + var body struct { + Proof struct { + Persisted bool `json:"persisted"` + ErrorCode string `json:"error_code"` + } `json:"proof"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body)) + assert.False(t, body.Proof.Persisted) + assert.Equal(t, "record_failed", body.Proof.ErrorCode) +} + +func TestEvidenceRequiredFromEnv(t *testing.T) { + for in, want := range map[string]bool{"": false, "false": false, " true ": true, "true": true} { + t.Setenv("CIC_EVIDENCE_REQUIRED", in) + got, err := evidenceRequiredFromEnv() + require.NoError(t, err, "value %q", in) + assert.Equal(t, want, got, "value %q", in) + } + for _, bad := range []string{"1", "yes", "TRUE", "tru"} { + t.Setenv("CIC_EVIDENCE_REQUIRED", bad) + _, err := evidenceRequiredFromEnv() + assert.Error(t, err, "value %q must be rejected, not silently treated as false", bad) + } +} diff --git a/context/SYMBOLS.md b/context/SYMBOLS.md index 02964be3..233d07fe 100644 --- a/context/SYMBOLS.md +++ b/context/SYMBOLS.md @@ -10,6 +10,8 @@ _*This file is automatically generated. Do not edit manually.*_ - `const KindWf` - `const KindModule` - `const MaxWASMModuleSize` +- `const ProofErrRecorderNotConfigured` +- `const ProofErrRecordFailed` ### Variables @@ -17,6 +19,7 @@ _*This file is automatically generated. Do not edit manually.*_ - `var ErrConflict` - `var ErrNotFound` - `var ErrTimeout` +- `var ErrEvidenceRequired` ### Functions @@ -24,6 +27,7 @@ _*This file is automatically generated. Do not edit manually.*_ - `func ComputeChainHashV1` - `func CoreEcho` - `func IsConflict` +- `func IsEvidenceRequired` - `func IsInvalid` - `func IsNotFound` - `func IsTimeout` @@ -41,6 +45,7 @@ _*This file is automatically generated. Do not edit manually.*_ - `type Cabinet` - `type CabinetOption` - `type CicWasmHost` +- `type CommitRecorder` - `type ExecutionGraph` - `type ExecutionGraphStep` - `type ExecutionRecorder` @@ -51,6 +56,7 @@ _*This file is automatically generated. Do not edit manually.*_ - `type ModuleID` - `type ModuleStore` - `type NativeFunctions` +- `type ProofStatus` - `type ProofTrace` - `type ProofTraceStep` - `type Schema` diff --git a/core/cabinet/api_set.go b/core/cabinet/api_set.go index 1ed4d3a6..5c43e059 100644 --- a/core/cabinet/api_set.go +++ b/core/cabinet/api_set.go @@ -5,11 +5,28 @@ import ( "fmt" ) +// Proof error codes reported in ProofStatus.ErrorCode. They carry no error text: +// the cause (which may name Vault paths or tokens) stays in the relay log. +const ( + ProofErrRecorderNotConfigured = "recorder_not_configured" + ProofErrRecordFailed = "record_failed" +) + +// ProofStatus tells the caller whether the ProofTrace of an execution was persisted. +// Execution succeeded, the chain hash was computed (SetResponse.Trace) and the record +// was persisted are three separate facts; this is the third. +type ProofStatus struct { + Persisted bool `json:"persisted"` + Commit string `json:"commit,omitempty"` // set when the recorder names the persisted record + ErrorCode string `json:"error_code,omitempty"` // set when Persisted is false +} + // SetResponse is the response to a Set call. type SetResponse struct { Results map[string]any Warnings []string Trace *ProofTrace + Proof *ProofStatus `json:"proof"` } // ValidateSet performs minimal structural validation of SetPayload and checks if referenced diff --git a/core/cabinet/error.go b/core/cabinet/error.go index 8072c73c..d86fdefb 100644 --- a/core/cabinet/error.go +++ b/core/cabinet/error.go @@ -16,6 +16,10 @@ var ( ErrConflict = errors.New("conflict") // conflicting resource state ErrNotFound = errors.New("not_found") // resource not found ErrTimeout = errors.New("timeout") // execution exceeded time budget + + // ErrEvidenceRequired: the workflow ran, but the ProofTrace was not persisted + // and the service is configured to require persisted evidence. + ErrEvidenceRequired = errors.New("evidence_required") ) // IsInvalid checks if an error is an ErrInvalid validation or integrity failure. @@ -30,6 +34,10 @@ func IsNotFound(err error) bool { return errors.Is(err, ErrNotFound) } // IsTimeout checks if an error is an ErrTimeout execution exceeded time budget. func IsTimeout(err error) bool { return errors.Is(err, ErrTimeout) } +// IsEvidenceRequired checks if an error is an ErrEvidenceRequired: the execution +// happened, but its proof was not persisted although the service requires it. +func IsEvidenceRequired(err error) bool { return errors.Is(err, ErrEvidenceRequired) } + // FieldError describes a validation error for a specific field. type FieldError struct { Field string `json:"field"` diff --git a/core/cabinet/execution_recorder.go b/core/cabinet/execution_recorder.go index 1a60492f..0b4acdd3 100644 --- a/core/cabinet/execution_recorder.go +++ b/core/cabinet/execution_recorder.go @@ -3,8 +3,16 @@ package cabinet import "context" // ExecutionRecorder is an optional sink for ProofTrace results produced by Setx. -// If nil, recording is silently skipped. Errors from recording are non-fatal: -// the caller logs a WARN and continues. +// If nil, recording is skipped. A recording error does not fail the call unless +// the service requires evidence (WithEvidenceRequired); either way the outcome is +// reported in SetResponse.Proof. type ExecutionRecorder interface { Record(ctx context.Context, workflowID string, trace interface{}) error } + +// CommitRecorder is an optional extension of ExecutionRecorder for recorders that +// can name the persisted record (for the git recorder: the commit hash). Setx +// prefers it, so the commit can be returned in SetResponse.Proof.Commit. +type CommitRecorder interface { + RecordCommit(ctx context.Context, workflowID string, trace interface{}) (commit string, err error) +} diff --git a/core/cabinet/proof_status_test.go b/core/cabinet/proof_status_test.go new file mode 100644 index 00000000..7e2ac95d --- /dev/null +++ b/core/cabinet/proof_status_test.go @@ -0,0 +1,153 @@ +package cabinet + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeRecorder is a plain ExecutionRecorder (no commit hash). +type fakeRecorder struct{ err error } + +func (f *fakeRecorder) Record(context.Context, string, interface{}) error { return f.err } + +// fakeCommitRecorder is an ExecutionRecorder that also names the persisted record. +type fakeCommitRecorder struct { + commit string + err error +} + +func (f *fakeCommitRecorder) Record(ctx context.Context, id string, tr interface{}) error { + _, err := f.RecordCommit(ctx, id, tr) + return err +} + +func (f *fakeCommitRecorder) RecordCommit(context.Context, string, interface{}) (string, error) { + return f.commit, f.err +} + +// newProofTestService builds a service with one native "echo" workflow. +func newProofTestService(t *testing.T, rec ExecutionRecorder, required bool) (*cabinetService, *SetPayload) { + t.Helper() + ctx := context.Background() + modStore := NewMemModuleStore() + wfStore := NewMemWorkflowStore() + + native := make(NativeFunctions) + native["echo"] = CoreEcho + mod, err := NewModuleDescriptor(ctx, BaseBlock{Name: "echo", Version: "1.0"}, "native", nil, nil, nil) + require.NoError(t, err) + mod.NativeImpl = native + modStore.Put(mod) + + wf, err := NewWorkflow(ctx, WorkflowIdentity{Version: "1.0"}, WorkflowConfig{Name: "wf.proof", Steps: []string{"echo@1.0.echo(input)"}}, WorkflowSigning{}) + require.NoError(t, err) + wfStore.Put(wf) + + svc := &cabinetService{ + modules: modStore, + workflows: wfStore, + wasmManager: new(MockWasmManager), + clock: func() string { return "2026-01-01T00:00:00Z" }, + recorder: rec, + evidenceRequired: required, + } + return svc, &SetPayload{WorkflowID: wf.ID(), Payload: map[string]any{"msg": "hello"}} +} + +func TestSetx_Proof_RecorderHealthy_PersistedWithCommit(t *testing.T) { + svc, p := newProofTestService(t, &fakeCommitRecorder{commit: "abc123"}, false) + resp, err := svc.Setx(context.Background(), p) + require.NoError(t, err) + require.NotNil(t, resp.Proof) + assert.True(t, resp.Proof.Persisted) + assert.Equal(t, "abc123", resp.Proof.Commit) + assert.Empty(t, resp.Proof.ErrorCode) + require.NotNil(t, resp.Trace) +} + +func TestSetx_Proof_PlainRecorderHealthy_PersistedWithoutCommit(t *testing.T) { + svc, p := newProofTestService(t, &fakeRecorder{}, false) + resp, err := svc.Setx(context.Background(), p) + require.NoError(t, err) + require.NotNil(t, resp.Proof) + assert.True(t, resp.Proof.Persisted) + assert.Empty(t, resp.Proof.Commit, "a recorder that cannot name the record returns no commit") +} + +// Recorder outage, evidence not required: the call succeeds, but the response says the +// proof was not persisted, with an error code, and keeps the trace (chain hash). +func TestSetx_Proof_RecorderOutage_NotRequired_ReportsNotPersisted(t *testing.T) { + for name, rec := range map[string]ExecutionRecorder{ + "plain": &fakeRecorder{err: errors.New("vault unavailable at secret/ci-token")}, + "commit": &fakeCommitRecorder{err: errors.New("vault unavailable at secret/ci-token")}, + } { + t.Run(name, func(t *testing.T) { + svc, p := newProofTestService(t, rec, false) + resp, err := svc.Setx(context.Background(), p) + require.NoError(t, err, "evidence is not required: the execution stays a success") + require.NotNil(t, resp.Proof, "the outcome must always be reported") + assert.False(t, resp.Proof.Persisted) + assert.Equal(t, ProofErrRecordFailed, resp.Proof.ErrorCode) + assert.Empty(t, resp.Proof.Commit) + require.NotNil(t, resp.Trace, "the trace is kept so partial evidence is not lost") + assert.NotEmpty(t, resp.Trace.ChainHash) + + raw, err := json.Marshal(resp) + require.NoError(t, err) + assert.NotContains(t, string(raw), "vault", "the recorder error text must not reach the response") + assert.NotContains(t, string(raw), "secret/ci-token") + assert.Contains(t, string(raw), `"proof":{"persisted":false,"error_code":"record_failed"}`) + }) + } +} + +// Recorder outage, evidence required: an explicit failure, never a plain success. +func TestSetx_Proof_RecorderOutage_Required_Fails(t *testing.T) { + svc, p := newProofTestService(t, &fakeCommitRecorder{err: errors.New("vault unavailable")}, true) + resp, err := svc.Setx(context.Background(), p) + require.Error(t, err) + assert.Nil(t, resp) + assert.True(t, IsEvidenceRequired(err)) + assert.Contains(t, err.Error(), "chain_hash=", "the chain hash must survive in the error") + assert.Contains(t, err.Error(), ProofErrRecordFailed) + assert.NotContains(t, err.Error(), "vault unavailable", "the recorder error text must not reach the caller") +} + +func TestSetx_Proof_NoRecorder(t *testing.T) { + t.Run("not required reports not persisted", func(t *testing.T) { + svc, p := newProofTestService(t, nil, false) + resp, err := svc.Setx(context.Background(), p) + require.NoError(t, err) + require.NotNil(t, resp.Proof) + assert.False(t, resp.Proof.Persisted) + assert.Equal(t, ProofErrRecorderNotConfigured, resp.Proof.ErrorCode) + }) + t.Run("required fails", func(t *testing.T) { + svc, p := newProofTestService(t, nil, true) + _, err := svc.Setx(context.Background(), p) + require.Error(t, err) + assert.True(t, IsEvidenceRequired(err)) + assert.Contains(t, err.Error(), ProofErrRecorderNotConfigured) + }) +} + +func TestSetx_Proof_Required_HealthyRecorder_Succeeds(t *testing.T) { + svc, p := newProofTestService(t, &fakeCommitRecorder{commit: "c0ffee"}, true) + resp, err := svc.Setx(context.Background(), p) + require.NoError(t, err) + assert.True(t, resp.Proof.Persisted) + assert.Equal(t, "c0ffee", resp.Proof.Commit) +} + +func TestWithEvidenceRequired_SetsTheField(t *testing.T) { + svc := &cabinetService{} + WithEvidenceRequired(true)(svc) + assert.True(t, svc.evidenceRequired) + WithEvidenceRequired(false)(svc) + assert.False(t, svc.evidenceRequired) +} diff --git a/core/cabinet/service.go b/core/cabinet/service.go index 1d2c34f2..8d400353 100644 --- a/core/cabinet/service.go +++ b/core/cabinet/service.go @@ -97,6 +97,11 @@ type cabinetService struct { wasmManager wasmManager clock func() string // injected; returns UTC timestamp for ProofTrace recorder ExecutionRecorder // optional; nil = skip recording + + // evidenceRequired makes an execution whose proof was not persisted (recording + // failed, or no recorder is configured) fail with ErrEvidenceRequired instead of + // returning a plain success. + evidenceRequired bool } // CabinetOption is a functional option for configuring a cabinetService. @@ -110,6 +115,16 @@ func WithRecorder(r ExecutionRecorder) CabinetOption { } } +// WithEvidenceRequired makes persisted evidence mandatory: when the ProofTrace of an +// execution is not persisted, Setx returns ErrEvidenceRequired (the message carries +// the chain hash) instead of a success. Off by default: a recording outage is then +// reported in SetResponse.Proof only. +func WithEvidenceRequired(required bool) CabinetOption { + return func(s *cabinetService) { + s.evidenceRequired = required + } +} + // WithObsSinks wires observability sinks into the underlying WasmManager. // Both arguments are nil-safe: passing nil disables the corresponding emissions. func WithObsSinks(logSink obs.LogSink, metricSink obs.MetricSink) CabinetOption { @@ -295,23 +310,49 @@ func (c *cabinetService) Setx(ctx context.Context, in *SetPayload) (*SetResponse Timestamp: clockFn(), } - // 4. Record the execution (non-fatal: Vault may be unavailable). - if c.recorder != nil { - if err := c.recorder.Record(ctx, workflow.ID(), trace); err != nil { - logrus.WithFields(logrus.Fields{ - "workflow_id": workflow.ID(), - "error": err, - }).Warn("ProofTrace recording failed, continuing") - } + // 4. Record the execution. A failure is non-fatal by default (Vault may be + // unavailable) but is always reported in the response; with evidenceRequired it + // fails the call. The workflow has already run at this point, so the error + // carries the chain hash: the partial evidence is not lost. + proof := c.recordProof(ctx, workflow.ID(), trace) + if c.evidenceRequired && !proof.Persisted { + return nil, fmt.Errorf("%w: workflow %q executed but its proof was not persisted (%s), chain_hash=%s", + ErrEvidenceRequired, workflow.ID(), proof.ErrorCode, trace.ChainHash) } return &SetResponse{ Results: execCtx.variables, Warnings: nil, Trace: trace, + Proof: proof, }, nil } +// recordProof persists the trace through the configured recorder and reports the outcome. +func (c *cabinetService) recordProof(ctx context.Context, workflowID string, trace *ProofTrace) *ProofStatus { + if c.recorder == nil { + logrus.WithField("workflow_id", workflowID).Warn("ProofTrace not persisted: no recorder configured") + return &ProofStatus{ErrorCode: ProofErrRecorderNotConfigured} + } + var ( + commit string + err error + ) + if cr, ok := c.recorder.(CommitRecorder); ok { + commit, err = cr.RecordCommit(ctx, workflowID, trace) + } else { + err = c.recorder.Record(ctx, workflowID, trace) + } + if err != nil { + logrus.WithFields(logrus.Fields{ + "workflow_id": workflowID, + "error": err, + }).Warn("ProofTrace recording failed") + return &ProofStatus{ErrorCode: ProofErrRecordFailed} + } + return &ProofStatus{Persisted: true, Commit: commit} +} + // Close closes the underlying WASM runtime and frees all associated resources. func (c *cabinetService) Close(ctx context.Context) error { return c.wasmManager.Close(ctx) diff --git a/core/nexus/recorder/workflow_recorder.go b/core/nexus/recorder/workflow_recorder.go index 23b8a9fe..04eefa1e 100644 --- a/core/nexus/recorder/workflow_recorder.go +++ b/core/nexus/recorder/workflow_recorder.go @@ -27,13 +27,19 @@ func NewWorkflowRecorder(sr StateRecorder) *WorkflowRecorder { // Record persists the ProofTrace via the underlying StateRecorder. // workflowID is used as the RunID; trace is stored as the actual state. func (w *WorkflowRecorder) Record(ctx context.Context, workflowID string, trace interface{}) error { + _, err := w.RecordCommit(ctx, workflowID, trace) + return err +} + +// RecordCommit is Record that also returns the persisted record's commit hash. It +// satisfies cabinet.CommitRecorder, so Setx can return the hash in the response. +func (w *WorkflowRecorder) RecordCommit(ctx context.Context, workflowID string, trace interface{}) (string, error) { if w.sr == nil { - return fmt.Errorf("WorkflowRecorder: underlying StateRecorder is nil") + return "", fmt.Errorf("WorkflowRecorder: underlying StateRecorder is nil") } expected := types.ExpectedState{ RunID: workflowID, Timestamp: time.Now().UTC(), } - _, err := w.sr.RecordState(ctx, expected, trace) - return err + return w.sr.RecordState(ctx, expected, trace) } diff --git a/core/nexus/recorder/workflow_recorder_commit_test.go b/core/nexus/recorder/workflow_recorder_commit_test.go new file mode 100644 index 00000000..e4d92636 --- /dev/null +++ b/core/nexus/recorder/workflow_recorder_commit_test.go @@ -0,0 +1,67 @@ +package recorder + +import ( + "context" + "errors" + "os/exec" + "strings" + "testing" + + "centralrelay/core/cabinet" + "centralrelay/core/nexus/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type commitStateRecorder struct { + commit string + err error +} + +func (c *commitStateRecorder) RecordState(context.Context, types.ExpectedState, interface{}) (string, error) { + return c.commit, c.err +} + +// RecordCommit must hand the persisted record's commit hash to the caller. +func TestWorkflowRecorder_RecordCommit_ReturnsCommit(t *testing.T) { + w := NewWorkflowRecorder(&commitStateRecorder{commit: "abc123"}) + commit, err := w.RecordCommit(context.Background(), "wf@1.0", map[string]string{"k": "v"}) + require.NoError(t, err) + assert.Equal(t, "abc123", commit) +} + +func TestWorkflowRecorder_RecordCommit_Error(t *testing.T) { + w := NewWorkflowRecorder(&commitStateRecorder{err: errors.New("boom")}) + commit, err := w.RecordCommit(context.Background(), "wf@1.0", nil) + require.Error(t, err) + assert.Empty(t, commit) + + nilSR := NewWorkflowRecorder(nil) + _, err = nilSR.RecordCommit(context.Background(), "wf@1.0", nil) + require.Error(t, err) +} + +// WorkflowRecorder must stay a cabinet.CommitRecorder: Setx only returns the commit +// hash for recorders that implement it. Removing RecordCommit fails the build here. +var ( + _ cabinet.ExecutionRecorder = (*WorkflowRecorder)(nil) + _ cabinet.CommitRecorder = (*WorkflowRecorder)(nil) +) + +// With the real git recorder the returned commit is the actual new HEAD. +func TestWorkflowRecorder_RecordCommit_RealGitRecorder_ReturnsHead(t *testing.T) { + repo := t.TempDir() + sr, err := NewGitStateRecorder(repo, signingMock()) + require.NoError(t, err) + w := NewWorkflowRecorder(sr) + + commit, err := w.RecordCommit(context.Background(), "wf@1.0", map[string]string{"k": "v"}) + require.NoError(t, err) + + cmd := exec.Command("git", "rev-parse", "HEAD") + cmd.Dir = repo + out, err := cmd.Output() + require.NoError(t, err) + assert.Equal(t, strings.TrimSpace(string(out)), commit) + assert.Len(t, commit, 40) +} From deff69b9eb955740f30b70581437caa44bc1770e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sat, 14 Nov 2026 21:50:50 +0100 Subject: [PATCH 2/3] fix(cabinet): return the evidence-required state structurally; refresh MANIFEST.sha256 The required-evidence failure happens after the workflow has run, so a client must not read the 503 as a failure it can retry blindly. The error is now a structured EvidenceRequiredError (workflow id, chain hash, error code), and the 503 body adds execution_completed=true, proof.persisted=false and chain_hash next to the usual code/error. errors.Is(err, ErrEvidenceRequired) still holds. The previous commit regenerated context/SYMBOLS.md but not MANIFEST.sha256, which hashes it; manifest-check failed on that drift. MANIFEST.sha256 is regenerated with scripts/gen_manifest.sh and differs from the base only in the SYMBOLS.md line. Refs #165 --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEQCIAZePYkyY/K38oB2KVn/188gbMZ2g41xPmANwCLOtClSAiB6ZMAkahciOaKY4yAyzHyiTZb3PyI7iF0MQ7fnoZGYUA== hash-algorithm = sha256 digest = IKBez3zA0dxL5DHpfrwMq3HrBoz8gLKi3JY1Ctih7z8= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- MANIFEST.sha256 | 2 +- cmd/relay/main.go | 24 ++++++++++++++++- cmd/relay/main_api_test.go | 45 ++++++++++++++++++++++++++++--- context/SYMBOLS.md | 1 + core/cabinet/error.go | 17 ++++++++++++ core/cabinet/proof_status_test.go | 6 ++++- core/cabinet/service.go | 3 +-- 7 files changed, 89 insertions(+), 9 deletions(-) diff --git a/MANIFEST.sha256 b/MANIFEST.sha256 index 3f4c5f0b..a59dd7be 100644 --- a/MANIFEST.sha256 +++ b/MANIFEST.sha256 @@ -25,5 +25,5 @@ e2307e22b5e248d63e84f321bfc12c32912f3e492f01763414e7e0ca47bfe072 ./ai/ROADMAP_a a5faa116761dbccbb4e1569c8a300a99a7184bd0cc587bb3f429c10d44360237 ./context/GOALS.md 44424aa2a544a07bb3a888be9045d316b23067a1848339f2e3ce4db108103af2 ./context/LIMITS.md 7f0881c683fe2c1f06864d111ee54b238c0974704b64d2d3561c5c15507c58ba ./context/SYMBOLS.ALLOWPKGS -f7b749b4c1eedd9d3158d886d8614784530b68310f6ba192120907fe9d3bb140 ./context/SYMBOLS.md +c25f84395c9143dac68813ba1671433fe0c7636dfd5e676b40f985e6dab44965 ./context/SYMBOLS.md 1479b2a3cd99cc7f9ddd1b3bb58114840859a01ad9a406c62bf0a9e4fcd87f50 ./context/TESTING.md diff --git a/cmd/relay/main.go b/cmd/relay/main.go index 2620383a..bed0205c 100644 --- a/cmd/relay/main.go +++ b/cmd/relay/main.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -187,6 +188,27 @@ func writeError(w http.ResponseWriter, status int, code, msg string) { json.NewEncoder(w).Encode(map[string]string{"error": msg, "code": code}) } +// writeEvidenceRequired answers 503 for a workflow that ran but whose proof was not +// persisted while evidence is required. Besides the usual error/code it states +// structurally that the execution completed, so a client does not retry it blindly, +// and returns the chain hash of the executed workflow. +func writeEvidenceRequired(w http.ResponseWriter, err error) { + body := map[string]any{ + "error": err.Error(), + "code": "evidence_required", + "execution_completed": true, + "proof": cabinet.ProofStatus{Persisted: false}, + } + var evErr *cabinet.EvidenceRequiredError + if errors.As(err, &evErr) { + body["chain_hash"] = evErr.ChainHash + body["proof"] = cabinet.ProofStatus{Persisted: false, ErrorCode: evErr.ErrorCode} + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + json.NewEncoder(w).Encode(body) +} + // newMux creates a new HTTP multiplexer and registers all relay API routes. func newMux(s *relayServer) *http.ServeMux { mux := http.NewServeMux() @@ -354,7 +376,7 @@ func (s *relayServer) setHandler(w http.ResponseWriter, r *http.Request) { // The workflow ran, but its proof was not persisted and evidence is required. traceStatus = "error" logEntry.WithField("outcome", "evidence_required").WithError(err).Error("Workflow executed but its proof was not persisted") - writeError(w, http.StatusServiceUnavailable, "evidence_required", err.Error()) + writeEvidenceRequired(w, err) return } if cabinet.IsInvalid(err) { diff --git a/cmd/relay/main_api_test.go b/cmd/relay/main_api_test.go index c9e9581a..cbe5084f 100644 --- a/cmd/relay/main_api_test.go +++ b/cmd/relay/main_api_test.go @@ -672,12 +672,14 @@ func TestSetHandler_SourceDigestFromTreeHash(t *testing.T) { } // TestSetHandler_EvidenceRequired: the workflow ran, but its proof was not persisted -// and evidence is required — an explicit 503, never a plain 200. +// and evidence is required — an explicit 503, never a plain 200. The body states +// structurally that the execution completed (so a client does not retry it blindly), +// that the proof was not persisted, and carries the chain hash. func TestSetHandler_EvidenceRequired(t *testing.T) { mockAPI := new(mockCabinetService) mockAPI.On("GetWorkflow", mock.Anything).Return(&cabinet.Workflow{}, nil) mockAPI.On("Setx", mock.Anything, mock.Anything).Return(nil, - fmt.Errorf("%w: workflow \"any\" executed but its proof was not persisted (record_failed), chain_hash=deadbeef", cabinet.ErrEvidenceRequired)) + &cabinet.EvidenceRequiredError{WorkflowID: "any", ChainHash: "deadbeef", ErrorCode: cabinet.ProofErrRecordFailed}) logger := logrus.New() logger.SetOutput(new(bytes.Buffer)) @@ -690,8 +692,43 @@ func TestSetHandler_EvidenceRequired(t *testing.T) { server.setHandler(w, req) assert.Equal(t, http.StatusServiceUnavailable, w.Code) - assertErrorJSON(t, w, "evidence_required") - assert.Contains(t, w.Body.String(), "chain_hash=deadbeef", "the chain hash must reach the caller") + assert.Equal(t, "application/json", w.Header().Get("Content-Type")) + var body struct { + Code string `json:"code"` + Error string `json:"error"` + ExecutionCompleted bool `json:"execution_completed"` + ChainHash string `json:"chain_hash"` + Proof struct { + Persisted bool `json:"persisted"` + ErrorCode string `json:"error_code"` + } `json:"proof"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body)) + assert.Equal(t, "evidence_required", body.Code) + assert.NotEmpty(t, body.Error) + assert.True(t, body.ExecutionCompleted, "the workflow ran: the call must not look like a retry-safe failure") + assert.Equal(t, "deadbeef", body.ChainHash) + assert.False(t, body.Proof.Persisted) + assert.Equal(t, cabinet.ProofErrRecordFailed, body.Proof.ErrorCode) +} + +// A wrapped evidence error without the structured type still maps to the same 503. +func TestSetHandler_EvidenceRequired_Unstructured(t *testing.T) { + mockAPI := new(mockCabinetService) + mockAPI.On("GetWorkflow", mock.Anything).Return(&cabinet.Workflow{}, nil) + mockAPI.On("Setx", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("%w: no detail", cabinet.ErrEvidenceRequired)) + + logger := logrus.New() + logger.SetOutput(new(bytes.Buffer)) + server := &relayServer{api: mockAPI, config: &config{MaxJSONInputSize: 1024}, logger: logger} + + req := httptest.NewRequest(http.MethodPost, "/set", bytes.NewReader([]byte(`{"workflow_id":"any"}`))) + w := httptest.NewRecorder() + server.setHandler(w, req) + + assert.Equal(t, http.StatusServiceUnavailable, w.Code) + assert.Contains(t, w.Body.String(), `"execution_completed":true`) + assert.NotContains(t, w.Body.String(), "chain_hash") } // TestSetHandler_ProofInResponse: a successful execution reports the proof outcome. diff --git a/context/SYMBOLS.md b/context/SYMBOLS.md index 233d07fe..07c9707e 100644 --- a/context/SYMBOLS.md +++ b/context/SYMBOLS.md @@ -46,6 +46,7 @@ _*This file is automatically generated. Do not edit manually.*_ - `type CabinetOption` - `type CicWasmHost` - `type CommitRecorder` +- `type EvidenceRequiredError` - `type ExecutionGraph` - `type ExecutionGraphStep` - `type ExecutionRecorder` diff --git a/core/cabinet/error.go b/core/cabinet/error.go index d86fdefb..fd05b78d 100644 --- a/core/cabinet/error.go +++ b/core/cabinet/error.go @@ -34,6 +34,23 @@ func IsNotFound(err error) bool { return errors.Is(err, ErrNotFound) } // IsTimeout checks if an error is an ErrTimeout execution exceeded time budget. func IsTimeout(err error) bool { return errors.Is(err, ErrTimeout) } +// EvidenceRequiredError is the structured form of ErrEvidenceRequired. The workflow has +// already run when it is returned, so the caller must not treat the call as a plain +// failure it can safely retry; the fields carry the partial evidence. +type EvidenceRequiredError struct { + WorkflowID string + ChainHash string // the computed chain hash of the executed workflow + ErrorCode string // ProofErrRecordFailed or ProofErrRecorderNotConfigured +} + +func (e *EvidenceRequiredError) Error() string { + return fmt.Sprintf("%s: workflow %q executed but its proof was not persisted (%s), chain_hash=%s", + ErrEvidenceRequired, e.WorkflowID, e.ErrorCode, e.ChainHash) +} + +// Is makes errors.Is(err, ErrEvidenceRequired) hold for the structured error. +func (e *EvidenceRequiredError) Is(target error) bool { return target == ErrEvidenceRequired } + // IsEvidenceRequired checks if an error is an ErrEvidenceRequired: the execution // happened, but its proof was not persisted although the service requires it. func IsEvidenceRequired(err error) bool { return errors.Is(err, ErrEvidenceRequired) } diff --git a/core/cabinet/proof_status_test.go b/core/cabinet/proof_status_test.go index 7e2ac95d..4f149bbb 100644 --- a/core/cabinet/proof_status_test.go +++ b/core/cabinet/proof_status_test.go @@ -113,7 +113,11 @@ func TestSetx_Proof_RecorderOutage_Required_Fails(t *testing.T) { require.Error(t, err) assert.Nil(t, resp) assert.True(t, IsEvidenceRequired(err)) - assert.Contains(t, err.Error(), "chain_hash=", "the chain hash must survive in the error") + var evErr *EvidenceRequiredError + require.True(t, errors.As(err, &evErr), "the failure must be the structured error") + assert.NotEmpty(t, evErr.ChainHash, "the chain hash must survive in the error") + assert.Equal(t, ProofErrRecordFailed, evErr.ErrorCode) + assert.Contains(t, err.Error(), "chain_hash="+evErr.ChainHash) assert.Contains(t, err.Error(), ProofErrRecordFailed) assert.NotContains(t, err.Error(), "vault unavailable", "the recorder error text must not reach the caller") } diff --git a/core/cabinet/service.go b/core/cabinet/service.go index 8d400353..de08cefa 100644 --- a/core/cabinet/service.go +++ b/core/cabinet/service.go @@ -316,8 +316,7 @@ func (c *cabinetService) Setx(ctx context.Context, in *SetPayload) (*SetResponse // carries the chain hash: the partial evidence is not lost. proof := c.recordProof(ctx, workflow.ID(), trace) if c.evidenceRequired && !proof.Persisted { - return nil, fmt.Errorf("%w: workflow %q executed but its proof was not persisted (%s), chain_hash=%s", - ErrEvidenceRequired, workflow.ID(), proof.ErrorCode, trace.ChainHash) + return nil, &EvidenceRequiredError{WorkflowID: workflow.ID(), ChainHash: trace.ChainHash, ErrorCode: proof.ErrorCode} } return &SetResponse{ From 789bcdb66062d2ed76fdcacca02a06520d1d798a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Z=20Sink=C3=B3?= Date: Sun, 15 Nov 2026 08:32:15 +0100 Subject: [PATCH 3/3] docs(relay): document CIC_EVIDENCE_REQUIRED and the retry hazard of 503 evidence_required The switch is part of the PR's acceptance: it is a deployment setting with a safety consequence. The env tables now state its values (true/false, anything else is a start-up error), that a 503 evidence_required means the workflow has already run (execution_completed, proof.persisted=false, chain_hash), and that it must not be retried automatically because a retry executes the workflow again. Refs #165 --- [signing-metadata] key = cic-my-sign-key signature = vault:v1:MEUCIQCr8ddfjb/Mb1dpDOgy2Xgx2wszObTanBVuTksSgGwpUQIgYky7w2Rg2Co8AG3zIAeKFIAwikLOS8j3ofqAYW/vNWQ= hash-algorithm = sha256 digest = 7mQ7ZTZ5Po6JoOctCtOqjM3AOlKbQlgGYPn3yMgMtmA= [certificate] -----BEGIN CERTIFICATE----- MIICBjCCAaygAwIBAgIUSnRMR6RPnEbg296XWPOqq/u5PCwwCgYIKoZIzj0EAwIw QzELMAkGA1UEBhMCSFUxGTAXBgNVBAoMEENlbnRyYWxJbmZyYUNvcmUxGTAXBgNV BAMMEENJQyBEZXZlbG9wZXIgQ0EwHhcNMjYwMzIwMTMyMjU5WhcNMjYxMjMxMTMy MjU5WjBFMQswCQYDVQQGEwJIVTEZMBcGA1UECgwQQ2VudHJhbEluZnJhQ29yZTEb MBkGA1UEAwwSR2Fib3IgWm9sdGFuIFNpbmtvMFkwEwYHKoZIzj0CAQYIKoZIzj0D AQcDQgAEIG2CVmTfmLB9pLLclj7YmP2eedAjklpy4LGrU2ijoiy6Xqpuybv7OgJe i+ez31s65NEV8+X/ByeX1cstR988z6N8MHowCQYDVR0TBAIwADAdBgNVHQ4EFgQU yZN6AIX/TNnIJ9GwAa/NRN3ujHAwHwYDVR0jBBgwFoAUXn6CHYzPUqU4JVP8g+OS WeDYjhcwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEF BQcDBDAKBggqhkjOPQQDAgNIADBFAiEA+bFzXRoJ4PCQbhAAtpkcMjt0vNj5rEW0 lOMBGDNyaWkCIB1vmM7PcZzv/c9bIrxF5kqv6QXomouhByUfeNUTbpKW -----END CERTIFICATE----- --- docs/en/pipeline/relay-container.md | 1 + docs/hu/pipeline/relay-container.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/en/pipeline/relay-container.md b/docs/en/pipeline/relay-container.md index 2a9d71bc..66a6e58f 100644 --- a/docs/en/pipeline/relay-container.md +++ b/docs/en/pipeline/relay-container.md @@ -89,6 +89,7 @@ The `relay` service reads these from the environment (a `.env` file next to | `CIC_VAULT_MOUNT` | `transit` | Vault transit mount | | `CIC_VAULT_KEY` | — | Signing key; the token must be *authorised for this key* | | `CIC_TRUST_MODE` | — | PKI trust mode (`dev` or `prod`); **required**, no default, an empty value is a start-up error | +| `CIC_EVIDENCE_REQUIRED` | `false` | `true` or `false` (anything else is a start-up error). `true`: when the ProofTrace of a `/set` run was not persisted (recording failed, or no recorder is configured), the call returns **503 `evidence_required`**. **The workflow has already run:** the body says `execution_completed: true`, `proof.persisted: false` and carries the `chain_hash`. **Do not retry it automatically**: a retry executes the workflow again. Default: the call stays a success and `proof.persisted` reports the recording | ### `CIC_BUILD_DIR` must be shared with the builder container diff --git a/docs/hu/pipeline/relay-container.md b/docs/hu/pipeline/relay-container.md index c4f098ef..4feff3eb 100644 --- a/docs/hu/pipeline/relay-container.md +++ b/docs/hu/pipeline/relay-container.md @@ -90,6 +90,7 @@ A `relay` szolgáltatás ezeket olvassa a környezetből (a `docker-compose.yml` | `CIC_VAULT_MOUNT` | `transit` | Vault transit mount | | `CIC_VAULT_KEY` | — | Aláíró kulcs; a tokennek **erre a kulcsra** kell jogosultnak lennie | | `CIC_TRUST_MODE` | — | PKI trust mód (`dev` vagy `prod`); **kötelező**, nincs alapértelmezés, üres érték indulási hiba | +| `CIC_EVIDENCE_REQUIRED` | `false` | `true` vagy `false` (bármi más indulási hiba). `true`: ha egy `/set` futás ProofTrace-e nem rögzült (a rögzítés hibázott, vagy nincs recorder), a hívás **503 `evidence_required`**-et ad. **A workflow ekkor már lefutott:** a törzs `execution_completed: true`, `proof.persisted: false` és a `chain_hash` mezőt adja. **Ne próbáld újra automatikusan**: az újrapróbálás újra végrehajtja a workflow-t. Alapértelmezés: a hívás sikeres marad, a `proof.persisted` mutatja a rögzítést | ### A `CIC_BUILD_DIR`-t meg kell osztani a builder konténerrel