feat(cabinet): report whether the ProofTrace was persisted, and allow requiring it

MANIFEST.sha256

@@ -25,5 +25,5 @@ e2307e22b5e248d63e84f321bfc12c32912f3e492f01763414e7e0ca47bfe072  ./ai/ROADMAP_a
 a5faa116761dbccbb4e1569c8a300a99a7184bd0cc587bb3f429c10d44360237  ./context/GOALS.md
 44424aa2a544a07bb3a888be9045d316b23067a1848339f2e3ce4db108103af2  ./context/LIMITS.md
 7f0881c683fe2c1f06864d111ee54b238c0974704b64d2d3561c5c15507c58ba  ./context/SYMBOLS.ALLOWPKGS
-f7b749b4c1eedd9d3158d886d8614784530b68310f6ba192120907fe9d3bb140  ./context/SYMBOLS.md
+c25f84395c9143dac68813ba1671433fe0c7636dfd5e676b40f985e6dab44965  ./context/SYMBOLS.md
 1479b2a3cd99cc7f9ddd1b3bb58114840859a01ad9a406c62bf0a9e4fcd87f50  ./context/TESTING.md

cmd/relay/main.go

@@ -3,11 +3,13 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/google/uuid"
@@ -186,6 +188,27 @@ func writeError(w http.ResponseWriter, status int, code, msg string) {
 	json.NewEncoder(w).Encode(map[string]string{"error": msg, "code": code})
 }
 
+// writeEvidenceRequired answers 503 for a workflow that ran but whose proof was not
+// persisted while evidence is required. Besides the usual error/code it states
+// structurally that the execution completed, so a client does not retry it blindly,
+// and returns the chain hash of the executed workflow.
+func writeEvidenceRequired(w http.ResponseWriter, err error) {
+	body := map[string]any{
+		"error":               err.Error(),
+		"code":                "evidence_required",
+		"execution_completed": true,
+		"proof":               cabinet.ProofStatus{Persisted: false},
+	}
+	var evErr *cabinet.EvidenceRequiredError
+	if errors.As(err, &evErr) {
+		body["chain_hash"] = evErr.ChainHash
+		body["proof"] = cabinet.ProofStatus{Persisted: false, ErrorCode: evErr.ErrorCode}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusServiceUnavailable)
+	json.NewEncoder(w).Encode(body)
+}
+
 // newMux creates a new HTTP multiplexer and registers all relay API routes.
 func newMux(s *relayServer) *http.ServeMux {
 	mux := http.NewServeMux()
@@ -349,6 +372,13 @@ func (s *relayServer) setHandler(w http.ResponseWriter, r *http.Request) {
 			writeError(w, http.StatusGatewayTimeout, "timeout", "Workflow execution timed out: "+err.Error())
 			return
 		}
+		if cabinet.IsEvidenceRequired(err) {
+			// The workflow ran, but its proof was not persisted and evidence is required.
+			traceStatus = "error"
+			logEntry.WithField("outcome", "evidence_required").WithError(err).Error("Workflow executed but its proof was not persisted")
+			writeEvidenceRequired(w, err)
+			return
+		}
 		if cabinet.IsInvalid(err) {
 			traceStatus = "rejected"
 			logEntry.WithField("outcome", "invalid_request").WithError(err).Warn("Invalid request")
@@ -458,6 +488,19 @@ func loadSecretsFromVault(logger *logrus.Logger) *VaultSecrets {
 	}
 }
 
+// evidenceRequiredFromEnv reads CIC_EVIDENCE_REQUIRED. Unset or empty means false; only
+// "true" and "false" are accepted, so a typo cannot silently switch a required mode off.
+func evidenceRequiredFromEnv() (bool, error) {
+	switch v := strings.TrimSpace(os.Getenv("CIC_EVIDENCE_REQUIRED")); v {
+	case "", "false":
+		return false, nil
+	case "true":
+		return true, nil
+	default:
+		return false, fmt.Errorf("invalid CIC_EVIDENCE_REQUIRED %q (valid: \"true\", \"false\")", v)
+	}
+}
+
 // buildRecorderOption constructs a cabinet.WithRecorder option from the provided VaultSecrets.
 // If secrets is nil (Vault not configured), returns a no-op option.
 func buildRecorderOption(logger *logrus.Logger, secrets *VaultSecrets) cabinet.CabinetOption {
@@ -625,6 +668,13 @@ func main() {
 	}
 	secrets := loadSecretsFromVault(logger)
 	recorderOpt := buildRecorderOption(logger, secrets)
+	evidenceRequired, err := evidenceRequiredFromEnv()
+	if err != nil {
+		logger.WithError(err).Fatal("FATAL: invalid evidence configuration")
+	}
+	if evidenceRequired {
+		logger.Info("Evidence required: a workflow whose ProofTrace is not persisted fails with 503 evidence_required")
+	}
 
 	// Build artifact signer for cic.artifact.sign — uses the same Vault key as the recorder.
 	// Nil if Vault is not configured; cic_sign will be "unavailable" in that case (non-fatal).
@@ -678,7 +728,7 @@ func main() {
 	metricSink := obs.NewMultiMetricSink(logrusSink, promSink)
 	gitOps.SetObsSinks(logrusSink, metricSink) // relay.git.op_duration_ms + WARN logok
 	obsOpt := cabinet.WithObsSinks(logrusSink, metricSink)
-	cabinetSvc, err := cabinet.NewCabinetService(context.Background(), schemaStore, moduleStore, workflowStore, wasmConfig, recorderOpt, obsOpt)
+	cabinetSvc, err := cabinet.NewCabinetService(context.Background(), schemaStore, moduleStore, workflowStore, wasmConfig, recorderOpt, cabinet.WithEvidenceRequired(evidenceRequired), obsOpt)
 	if err != nil {
 		logger.WithField("error", err).Fatal("FATAL: Failed to create cabinet service")
 	}

cmd/relay/main_api_test.go

@@ -670,3 +670,108 @@ func TestSetHandler_SourceDigestFromTreeHash(t *testing.T) {
 	server.setHandler(w, req)
 	mockAPI.AssertExpectations(t)
 }
+
+// TestSetHandler_EvidenceRequired: the workflow ran, but its proof was not persisted
+// and evidence is required — an explicit 503, never a plain 200. The body states
+// structurally that the execution completed (so a client does not retry it blindly),
+// that the proof was not persisted, and carries the chain hash.
+func TestSetHandler_EvidenceRequired(t *testing.T) {
+	mockAPI := new(mockCabinetService)
+	mockAPI.On("GetWorkflow", mock.Anything).Return(&cabinet.Workflow{}, nil)
+	mockAPI.On("Setx", mock.Anything, mock.Anything).Return(nil,
+		&cabinet.EvidenceRequiredError{WorkflowID: "any", ChainHash: "deadbeef", ErrorCode: cabinet.ProofErrRecordFailed})
+
+	logger := logrus.New()
+	logger.SetOutput(new(bytes.Buffer))
+	server := &relayServer{api: mockAPI, config: &config{MaxJSONInputSize: 1024}, logger: logger}
+
+	req := httptest.NewRequest(http.MethodPost, "/set", bytes.NewReader([]byte(`{"workflow_id":"any"}`)))
+	req.Header.Set("Content-Type", "application/json")
+	w := httptest.NewRecorder()
+
+	server.setHandler(w, req)
+
+	assert.Equal(t, http.StatusServiceUnavailable, w.Code)
+	assert.Equal(t, "application/json", w.Header().Get("Content-Type"))
+	var body struct {
+		Code               string `json:"code"`
+		Error              string `json:"error"`
+		ExecutionCompleted bool   `json:"execution_completed"`
+		ChainHash          string `json:"chain_hash"`
+		Proof              struct {
+			Persisted bool   `json:"persisted"`
+			ErrorCode string `json:"error_code"`
+		} `json:"proof"`
+	}
+	require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body))
+	assert.Equal(t, "evidence_required", body.Code)
+	assert.NotEmpty(t, body.Error)
+	assert.True(t, body.ExecutionCompleted, "the workflow ran: the call must not look like a retry-safe failure")
+	assert.Equal(t, "deadbeef", body.ChainHash)
+	assert.False(t, body.Proof.Persisted)
+	assert.Equal(t, cabinet.ProofErrRecordFailed, body.Proof.ErrorCode)
+}
+
+// A wrapped evidence error without the structured type still maps to the same 503.
+func TestSetHandler_EvidenceRequired_Unstructured(t *testing.T) {
+	mockAPI := new(mockCabinetService)
+	mockAPI.On("GetWorkflow", mock.Anything).Return(&cabinet.Workflow{}, nil)
+	mockAPI.On("Setx", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("%w: no detail", cabinet.ErrEvidenceRequired))
+
+	logger := logrus.New()
+	logger.SetOutput(new(bytes.Buffer))
+	server := &relayServer{api: mockAPI, config: &config{MaxJSONInputSize: 1024}, logger: logger}
+
+	req := httptest.NewRequest(http.MethodPost, "/set", bytes.NewReader([]byte(`{"workflow_id":"any"}`)))
+	w := httptest.NewRecorder()
+	server.setHandler(w, req)
+
+	assert.Equal(t, http.StatusServiceUnavailable, w.Code)
+	assert.Contains(t, w.Body.String(), `"execution_completed":true`)
+	assert.NotContains(t, w.Body.String(), "chain_hash")
+}
+
+// TestSetHandler_ProofInResponse: a successful execution reports the proof outcome.
+func TestSetHandler_ProofInResponse(t *testing.T) {
+	mockAPI := new(mockCabinetService)
+	mockAPI.On("GetWorkflow", mock.Anything).Return(&cabinet.Workflow{}, nil)
+	mockAPI.On("Setx", mock.Anything, mock.Anything).Return(&cabinet.SetResponse{
+		Results: map[string]any{"ok": true},
+		Proof:   &cabinet.ProofStatus{Persisted: false, ErrorCode: cabinet.ProofErrRecordFailed},
+	}, nil)
+
+	logger := logrus.New()
+	logger.SetOutput(new(bytes.Buffer))
+	server := &relayServer{api: mockAPI, config: &config{MaxJSONInputSize: 1024}, logger: logger}
+
+	req := httptest.NewRequest(http.MethodPost, "/set", bytes.NewReader([]byte(`{"workflow_id":"any"}`)))
+	req.Header.Set("Content-Type", "application/json")
+	w := httptest.NewRecorder()
+
+	server.setHandler(w, req)
+
+	assert.Equal(t, http.StatusOK, w.Code, "evidence not required: a successful execution keeps HTTP 200")
+	var body struct {
+		Proof struct {
+			Persisted bool   `json:"persisted"`
+			ErrorCode string `json:"error_code"`
+		} `json:"proof"`
+	}
+	require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body))
+	assert.False(t, body.Proof.Persisted)
+	assert.Equal(t, "record_failed", body.Proof.ErrorCode)
+}
+
+func TestEvidenceRequiredFromEnv(t *testing.T) {
+	for in, want := range map[string]bool{"": false, "false": false, " true ": true, "true": true} {
+		t.Setenv("CIC_EVIDENCE_REQUIRED", in)
+		got, err := evidenceRequiredFromEnv()
+		require.NoError(t, err, "value %q", in)
+		assert.Equal(t, want, got, "value %q", in)
+	}
+	for _, bad := range []string{"1", "yes", "TRUE", "tru"} {
+		t.Setenv("CIC_EVIDENCE_REQUIRED", bad)
+		_, err := evidenceRequiredFromEnv()
+		assert.Error(t, err, "value %q must be rejected, not silently treated as false", bad)
+	}
+}

context/SYMBOLS.md

@@ -10,20 +10,24 @@ _*This file is automatically generated. Do not edit manually.*_
 - `const KindWf`
 - `const KindModule`
 - `const MaxWASMModuleSize`
+- `const ProofErrRecorderNotConfigured`
+- `const ProofErrRecordFailed`
 
 ### Variables
 
 - `var ErrInvalid`
 - `var ErrConflict`
 - `var ErrNotFound`
 - `var ErrTimeout`
+- `var ErrEvidenceRequired`
 
 ### Functions
 
 - `func CanonicalizeSetPayload`
 - `func ComputeChainHashV1`
 - `func CoreEcho`
 - `func IsConflict`
+- `func IsEvidenceRequired`
 - `func IsInvalid`
 - `func IsNotFound`
 - `func IsTimeout`
@@ -41,6 +45,8 @@ _*This file is automatically generated. Do not edit manually.*_
 - `type Cabinet`
 - `type CabinetOption`
 - `type CicWasmHost`
+- `type CommitRecorder`
+- `type EvidenceRequiredError`
 - `type ExecutionGraph`
 - `type ExecutionGraphStep`
 - `type ExecutionRecorder`
@@ -51,6 +57,7 @@ _*This file is automatically generated. Do not edit manually.*_
 - `type ModuleID`
 - `type ModuleStore`
 - `type NativeFunctions`
+- `type ProofStatus`
 - `type ProofTrace`
 - `type ProofTraceStep`
 - `type Schema`

core/cabinet/api_set.go

@@ -5,11 +5,28 @@ import (
 	"fmt"
 )
 
+// Proof error codes reported in ProofStatus.ErrorCode. They carry no error text:
+// the cause (which may name Vault paths or tokens) stays in the relay log.
+const (
+	ProofErrRecorderNotConfigured = "recorder_not_configured"
+	ProofErrRecordFailed          = "record_failed"
+)
+
+// ProofStatus tells the caller whether the ProofTrace of an execution was persisted.
+// Execution succeeded, the chain hash was computed (SetResponse.Trace) and the record
+// was persisted are three separate facts; this is the third.
+type ProofStatus struct {
+	Persisted bool   `json:"persisted"`
+	Commit    string `json:"commit,omitempty"`     // set when the recorder names the persisted record
+	ErrorCode string `json:"error_code,omitempty"` // set when Persisted is false
+}
+
 // SetResponse is the response to a Set call.
 type SetResponse struct {
 	Results  map[string]any
 	Warnings []string
 	Trace    *ProofTrace
+	Proof    *ProofStatus `json:"proof"`
 }
 
 // ValidateSet performs minimal structural validation of SetPayload and checks if referenced

core/cabinet/error.go

@@ -16,6 +16,10 @@ var (
 	ErrConflict = errors.New("conflict")  // conflicting resource state
 	ErrNotFound = errors.New("not_found") // resource not found
 	ErrTimeout  = errors.New("timeout")   // execution exceeded time budget
+
+	// ErrEvidenceRequired: the workflow ran, but the ProofTrace was not persisted
+	// and the service is configured to require persisted evidence.
+	ErrEvidenceRequired = errors.New("evidence_required")
 )
 
 // IsInvalid checks if an error is an ErrInvalid validation or integrity failure.
@@ -30,6 +34,27 @@ func IsNotFound(err error) bool { return errors.Is(err, ErrNotFound) }
 // IsTimeout checks if an error is an ErrTimeout execution exceeded time budget.
 func IsTimeout(err error) bool { return errors.Is(err, ErrTimeout) }
 
+// EvidenceRequiredError is the structured form of ErrEvidenceRequired. The workflow has
+// already run when it is returned, so the caller must not treat the call as a plain
+// failure it can safely retry; the fields carry the partial evidence.
+type EvidenceRequiredError struct {
+	WorkflowID string
+	ChainHash  string // the computed chain hash of the executed workflow
+	ErrorCode  string // ProofErrRecordFailed or ProofErrRecorderNotConfigured
+}
+
+func (e *EvidenceRequiredError) Error() string {
+	return fmt.Sprintf("%s: workflow %q executed but its proof was not persisted (%s), chain_hash=%s",
+		ErrEvidenceRequired, e.WorkflowID, e.ErrorCode, e.ChainHash)
+}
+
+// Is makes errors.Is(err, ErrEvidenceRequired) hold for the structured error.
+func (e *EvidenceRequiredError) Is(target error) bool { return target == ErrEvidenceRequired }
+
+// IsEvidenceRequired checks if an error is an ErrEvidenceRequired: the execution
+// happened, but its proof was not persisted although the service requires it.
+func IsEvidenceRequired(err error) bool { return errors.Is(err, ErrEvidenceRequired) }
+
 // FieldError describes a validation error for a specific field.
 type FieldError struct {
 	Field string `json:"field"`

core/cabinet/execution_recorder.go

@@ -3,8 +3,16 @@ package cabinet
 import "context"
 
 // ExecutionRecorder is an optional sink for ProofTrace results produced by Setx.
-// If nil, recording is silently skipped. Errors from recording are non-fatal:
-// the caller logs a WARN and continues.
+// If nil, recording is skipped. A recording error does not fail the call unless
+// the service requires evidence (WithEvidenceRequired); either way the outcome is
+// reported in SetResponse.Proof.
 type ExecutionRecorder interface {
 	Record(ctx context.Context, workflowID string, trace interface{}) error
 }
+
+// CommitRecorder is an optional extension of ExecutionRecorder for recorders that
+// can name the persisted record (for the git recorder: the commit hash). Setx
+// prefers it, so the commit can be returned in SetResponse.Proof.Commit.
+type CommitRecorder interface {
+	RecordCommit(ctx context.Context, workflowID string, trace interface{}) (commit string, err error)
+}