From 6e906b49202d3a203f4febdd2c450b3b15b1731a Mon Sep 17 00:00:00 2001 From: tglemser <62476591+tglemser@users.noreply.github.com> Date: Fri, 20 Nov 2026 10:03:52 +0000 Subject: [PATCH 1/6] fix(agent): publish record_fact as a facts-only contract The schema the model sees for record_fact declared every fact field twice: at the top level (single-fact shape) and inside facts[] (batch shape), with the handler deciding which shape a call is. A model that fills every declared property sends the single fact inside facts[] and a copy, or a placeholder such as "", [] or "x", of each field beside it; since PR #282 every such call is refused by name, and the refusal text ("summary is not allowed at the top level") leads the model to send a dummy value next, so the same refusal repeats until the intent ends. Observed with a model over the OpenAI Responses API: 209 of 212 record_fact calls of two tasks were refused, 156 of them identical in one intent. The published schema is now {facts:[{...}]} and nothing else (required: facts, additionalProperties false). The handler keeps the strict contract of PR #282: unknown or mis-cased fields, invalid or foreign asset ids, missing summaries, empty batches and any top-level field beside facts (other than the legacy intent_id) are refused before anything is written, with an error that names the fields and the one accepted shape. It still accepts the legacy single fact and the legacy batch with a top-level intent_id. An item without intent_id is linked to the intent the worker is running (SetOwnerNode), as report_finding already does. record_fact opts out of the harness schema check (norma Spec.RawInput) so the stricter published schema does not reject the legacy shapes before the handler sees them; the catalog decorator forwards the opt-out. RepairRecordFactSchema republishes stored and cached schemas as facts-only and keeps the annotations of facts and its items. A one-time startup refresh replaces the stored description and the texts that referred to the top level when they equal the previous defaults; operator edits stay and are logged. seedFactSourceSchema adds the source enum on the items. Tests: the handler contract (TestRecordFactSummaryContract, TestRecordFactStrictFields, TestRecordFactBatchTopLevelConflicts, TestRecordFactRefusesTopLevelCopiesBesideFacts), the published schema and its repair (TestRecordFactSchemaIsFactsOnly, TestRecordFactStoredContractRepair, TestRecordFactInputSchemaUpgrade, TestRecordFactSourceSchemaUpgrade, TestWireTools), the running-intent default, the one-time refresh (TestRefreshRecordFactContract) and a synthetic replay of the call shapes (TestRecordFactShapesReplay). Co-Authored-By: Claude Mythos 5.1 --- agent/fact_assets_test.go | 7 +- agent/fact_input.go | 144 ++++++++++------- agent/fact_input_legacy.go | 36 +++++ agent/fact_input_test.go | 260 ++++++++++++++++++++++++++++-- agent/record_fact_shapes_test.go | 128 +++++++++++++++ agent/toolcatalog.go | 8 + agent/tools.go | 49 +++++- server/fact_input_schema.go | 79 +++++++++ server/fact_input_schema_test.go | 153 +++++++++++++++++- server/fact_source_schema_test.go | 27 +++- server/orchestration.go | 19 ++- server/tools_wire_test.go | 20 ++- 12 files changed, 822 insertions(+), 108 deletions(-) create mode 100644 agent/fact_input_legacy.go create mode 100644 agent/record_fact_shapes_test.go diff --git a/agent/fact_assets_test.go b/agent/fact_assets_test.go index d2b1a7838..ec67e0271 100644 --- a/agent/fact_assets_test.go +++ b/agent/fact_assets_test.go @@ -115,10 +115,9 @@ func TestRecordFactAssetAliases(t *testing.T) { func TestRecordFactAssetAliasSchema(t *testing.T) { schema := NewToolSet(nil, "").recordFact().InputSchema() props := schema["properties"].(map[string]any) - for _, name := range []string{"asset_ids", "tested_asset_ids"} { - if props[name] == nil { - t.Fatalf("single schema missing %s", name) - } + // Facts-only contract: the alias lives on the items. + if len(props) != 1 || props["facts"] == nil { + t.Fatalf("top level must hold facts only: %v", props) } item := props["facts"].(map[string]any)["items"].(map[string]any) itemProps, _ := item["properties"].(map[string]any) diff --git a/agent/fact_input.go b/agent/fact_input.go index 60c08902d..3b6581b8d 100644 --- a/agent/fact_input.go +++ b/agent/fact_input.go @@ -4,10 +4,12 @@ import ( "bytes" "encoding/json" "fmt" + "sort" + "strings" ) // Match strings.TrimSpace, including Unicode whitespace, in the published schema. -const factWhitespace = "\t\n\v\f\r \u0085\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000" +const factWhitespace = "\t\n\v\f\r \u0085   - 

   " func factAssetIDSchema() map[string]any { return map[string]any{ @@ -16,10 +18,13 @@ func factAssetIDSchema() map[string]any { } } +// factItemSchema is one fact: the shape of every facts[] item. It is also the +// shape of the legacy single-fact call (fields at the top level, no facts), +// which the handler still accepts but the published schema no longer shows. func factItemSchema() map[string]any { schema := obj(map[string]any{ - "summary": str("Required on every fact: a non-empty one-sentence summary of this conclusion. Batch items do not inherit summary from the top level."), - "intent_id": idp("The intent that produced this fact; only this field inherits from the top level in a batch."), + "summary": str("Required: a non-empty one-sentence summary of this conclusion."), + "intent_id": idp("The intent that produced this fact. Omit it to link the fact to the intent this worker is running; a batch may name a different intent per item."), "detail": str("Details of this fact, including all related observations."), "evidence": str("One line of key evidence: the command and the most proving output lines. Put longer output in detail."), "confidence": str("observed (seen directly in output) | inferred (inferred from phenomena). Label negative conclusions honestly."), @@ -34,66 +39,65 @@ func factItemSchema() map[string]any { return schema } +// recordFactFactsDescription documents the only top-level property of +// record_fact. The wording tells a model that tends to fill every declared +// property that there is nothing else to fill. +const recordFactFactsDescription = "The facts to record, usually exactly one. Every item is a complete fact with its own summary. This is the only top-level property: never send summary, detail, evidence, confidence, source, intent_id or asset ids beside facts. Returned ids match item order; retry only failed indices." + +// recordFactContractNote is appended to the record_fact description: the one +// call shape the model is given. +const recordFactContractNote = "\n★Call shape: exactly {\"facts\":[{...}]} with one item per distinct conclusion (usually one item). Each item carries its own summary, detail, evidence, confidence, source, asset_ids and, when it differs from the intent you are running, intent_id. Nothing else at the top level; a top-level fact field beside facts is refused and nothing is written." + +// RecordFactContractNote exposes the text to the server's one-time refresh. +func RecordFactContractNote() string { return recordFactContractNote } + +// recordFactSchema is the published input contract: facts[] and nothing else. +// Providers reject root-level anyOf, and the earlier contract (fact fields at +// the top level beside facts[]) made models fill both; the handler alone keeps +// the legacy shapes alive. func recordFactSchema() map[string]any { - props := factItemSchema()["properties"].(map[string]any) - props["facts"] = map[string]any{ - "type": "array", "minItems": 1, "items": factItemSchema(), - "description": "A non-empty batch of complete facts. Every item needs its own summary. Only intent_id inherits from the top level. Returned ids match item order; retry only failed indices.", - } - schema := obj(props) - // Providers reject root-level anyOf. The handler enforces single/batch mode. + schema := obj(map[string]any{ + "facts": map[string]any{ + "type": "array", "minItems": 1, "items": factItemSchema(), + "description": recordFactFactsDescription, + }, + }, "facts") schema["additionalProperties"] = false return schema } -// RepairRecordFactSchema restores the single/batch input contract in stored or -// cached schemas, preserving annotations only on fields the handler accepts. +// RepairRecordFactSchema brings a stored or cached record_fact schema to the +// facts-only contract. The structure always comes from the code; of the stored +// schema only the annotations (description, default) of facts and of its known +// item fields survive. Top-level fact fields of the earlier contract are +// dropped: the handler still accepts them, but the model must not see them. func RepairRecordFactSchema(stored map[string]any) map[string]any { - raw, _ := json.Marshal(stored) - var out map[string]any - _ = json.Unmarshal(raw, &out) - if out == nil { - out = map[string]any{} + out := recordFactSchema() + facts := out["properties"].(map[string]any)["facts"].(map[string]any) + storedProps, _ := stored["properties"].(map[string]any) + storedFacts, _ := storedProps["facts"].(map[string]any) + if storedFacts == nil { + return out } - mergeFactContract(out, recordFactSchema()) - // A batch requires summaries on its items, never a redundant top-level one. - delete(out, "required") - delete(out, "anyOf") - canonical := recordFactSchema()["properties"].(map[string]any) - props := out["properties"].(map[string]any) - for name := range props { - if _, known := canonical[name]; !known { - delete(props, name) - } + if d, ok := storedFacts["description"].(string); ok && strings.TrimSpace(d) != "" { + facts["description"] = d } - items := props["facts"].(map[string]any)["items"].(map[string]any) - itemProps := items["properties"].(map[string]any) - for name := range itemProps { - if _, known := canonical[name]; !known || name == "facts" { - delete(itemProps, name) - } - } - return out -} - -func mergeFactContract(dst, src map[string]any) { - for key, value := range src { - if key == "description" || key == "default" { - if _, ok := dst[key]; ok { - continue - } + storedItems, _ := storedFacts["items"].(map[string]any) + storedItemProps, _ := storedItems["properties"].(map[string]any) + itemProps := facts["items"].(map[string]any)["properties"].(map[string]any) + for name, p := range itemProps { + sp, _ := storedItemProps[name].(map[string]any) + if sp == nil { + continue } - if child, ok := value.(map[string]any); ok { - target, _ := dst[key].(map[string]any) - if target == nil { - target = map[string]any{} - dst[key] = target + prop := p.(map[string]any) + for _, key := range []string{"description", "default"} { + if v, ok := sp[key]; ok && v != nil { + prop[key] = v } - mergeFactContract(target, child) - } else { - dst[key] = value } } + return out } type recordFactInput struct { @@ -103,6 +107,9 @@ type recordFactInput struct { // Validate exact field names before decoding: encoding/json otherwise treats // Summary, SUMMARY, etc. as aliases even with DisallowUnknownFields enabled. +// Three input shapes reach the handler: the published one ({facts:[...]}), the +// legacy single fact (fields at the top level, no facts) and the legacy batch +// with a top-level intent_id as the default of its items. func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) { var out recordFactInput fields, err := exactFactFields(in, true) @@ -111,11 +118,6 @@ func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) { } _, batch := fields["facts"] if batch { - for name := range fields { - if name != "intent_id" && name != "facts" { - return out, true, fmt.Errorf("field %q is not allowed at the top level in batch mode; only intent_id and facts are accepted", name) - } - } var items []json.RawMessage if err := json.Unmarshal(fields["facts"], &items); err != nil { return out, true, fmt.Errorf("facts must be an array: %w", err) @@ -123,10 +125,16 @@ func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) { if len(items) == 0 { return out, true, fmt.Errorf("facts must be a non-empty array of complete facts, each with its own summary") } + itemFields := make([]map[string]json.RawMessage, len(items)) for i, item := range items { - if _, err := exactFactFields(item, false); err != nil { + f, err := exactFactFields(item, false) + if err != nil { return out, true, fmt.Errorf("facts[%d]: %w", i, err) } + itemFields[i] = f + } + if err := checkBatchTopLevel(fields, itemFields); err != nil { + return out, true, err } } decoder := json.NewDecoder(bytes.NewReader(in)) @@ -137,6 +145,28 @@ func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) { return out, batch, nil } +// checkBatchTopLevel enforces the contract at the top level of a facts[] call, +// where only facts and (legacy) intent_id carry meaning. Any other top-level +// field is refused by name before anything is written, including empty +// placeholders and copies of a fact's fields: a value there is at best +// redundant and at worst a second, conflicting claim. The message names the +// one accepted shape, so a model that filled every declared property of an +// older schema learns where the fields belong. +func checkBatchTopLevel(fields map[string]json.RawMessage, items []map[string]json.RawMessage) error { + var extra []string + for name := range fields { + if name == "intent_id" || name == "facts" { + continue + } + extra = append(extra, fmt.Sprintf("%q", name)) + } + if len(extra) == 0 { + return nil + } + sort.Strings(extra) + return fmt.Errorf("top-level field(s) %s are not allowed beside facts (%d item(s)): record_fact takes only {\"facts\":[...]}; put every field of a fact inside its facts item and send nothing else at the top level", strings.Join(extra, ", "), len(items)) +} + func exactFactFields(in json.RawMessage, topLevel bool) (map[string]json.RawMessage, error) { var fields map[string]json.RawMessage if err := json.Unmarshal(in, &fields); err != nil { diff --git a/agent/fact_input_legacy.go b/agent/fact_input_legacy.go new file mode 100644 index 000000000..769c53977 --- /dev/null +++ b/agent/fact_input_legacy.go @@ -0,0 +1,36 @@ +package agent + +// Texts of the earlier record_fact contract (fact fields at the top level +// beside facts[]), kept verbatim so the server's one-time refresh +// (server/fact_input_schema.go) can tell a stored default from an operator +// edit. Nothing here is published to a model. + +// recordFactLegacyDescription is the record_fact description seeded up to the +// facts-only contract. +const recordFactLegacyDescription = "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n" + + "⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n" + + "★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n" + + "⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n" + + " · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n" + + " · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n" + + " · source = where the content came from: target_response (content the target itself returned: page bodies, headers, error messages, banners, even when a tool fetched it) | tool_output (a tool's own result about the target: port states, scanner verdicts, status codes, timings) | third_party_intel (OSINT, CVE databases, public reports) | operator_input (given by the human operator) | inference (your own reasoning; the default when omitted). Any other value is refused.\n" + + " · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen." + +// recordFactLegacyItemDescriptions are the facts[] item descriptions of the +// earlier contract whose wording referred to the top level; the refresh +// replaces exactly these texts with the current ones. +var recordFactLegacyItemDescriptions = map[string]string{ + "summary": "Required on every fact: a non-empty one-sentence summary of this conclusion. Batch items do not inherit summary from the top level.", + "intent_id": "The intent that produced this fact; only this field inherits from the top level in a batch.", +} + +// recordFactLegacyFactsDescription is the facts property description of the +// earlier contract. +const recordFactLegacyFactsDescription = "A non-empty batch of complete facts. Every item needs its own summary. Only intent_id inherits from the top level. Returned ids match item order; retry only failed indices." + +// RecordFactLegacyDescriptions returns the description texts of the earlier +// contract that the refresh may replace: the tool description, the facts +// property description and the item descriptions. +func RecordFactLegacyDescriptions() (tool []string, facts string, items map[string]string) { + return []string{recordFactLegacyDescription}, recordFactLegacyFactsDescription, recordFactLegacyItemDescriptions +} diff --git a/agent/fact_input_test.go b/agent/fact_input_test.go index 6c02c128b..7b48a1356 100644 --- a/agent/fact_input_test.go +++ b/agent/fact_input_test.go @@ -3,9 +3,11 @@ package agent import ( "encoding/json" "regexp" + "strconv" "strings" "testing" + "github.com/Autumn-27/artex/db" actool "github.com/Autumn-27/norma/tool" ) @@ -47,6 +49,10 @@ func TestRecordFactSummaryContract(t *testing.T) { } } +// A stored schema of the earlier contract (fact fields at the top level beside +// facts[], operator annotations, a stray property) is republished as the +// facts-only contract; annotations of facts and of its items survive, the top +// level loses every fact field, and the result is idempotent. func TestRecordFactStoredContractRepair(t *testing.T) { old := map[string]any{ "type": "object", "required": []any{"summary"}, @@ -55,32 +61,40 @@ func TestRecordFactStoredContractRepair(t *testing.T) { "summary": map[string]any{"type": "string", "description": "operator summary"}, "confidence": map[string]any{"type": "string", "default": "inferred"}, "custom_field": map[string]any{"type": "string"}, - "facts": map[string]any{"type": "array", "items": map[string]any{"type": "object"}}, + "facts": map[string]any{"type": "array", "description": "operator facts", "items": map[string]any{"type": "object", "properties": map[string]any{ + "summary": map[string]any{"type": "string", "description": "operator item summary"}, + "confidence": map[string]any{"type": "string", "default": "observed"}, + "custom": map[string]any{"type": "string"}, + }}}, }, } base := NewToolSet(nil, "").recordFact() repaired := DecorateTool(base, "operator tool", old) schema := repaired.InputSchema() - if _, required := schema["required"]; required { - t.Fatal("batch still requires a top-level summary") + if req, _ := schema["required"].([]string); len(req) != 1 || req[0] != "facts" { + t.Fatalf("facts must be the only required property: %v", schema["required"]) } if _, exists := schema["anyOf"]; exists { t.Fatal("root anyOf reaches the provider") } props := schema["properties"].(map[string]any) - item := props["facts"].(map[string]any)["items"].(map[string]any) + if len(props) != 1 || props["facts"] == nil || schema["additionalProperties"] != false { + t.Fatalf("top level must hold facts only: %v", props) + } + facts := props["facts"].(map[string]any) + item := facts["items"].(map[string]any) required := item["required"].([]string) - if len(required) != 1 || required[0] != "summary" || props["facts"].(map[string]any)["minItems"] != 1 { + if len(required) != 1 || required[0] != "summary" || facts["minItems"] != 1 { t.Fatal("batch must require a nonempty list with a summary per item") } - if props["summary"].(map[string]any)["minLength"] != 1 || - item["properties"].(map[string]any)["summary"].(map[string]any)["minLength"] != 1 { + itemProps := item["properties"].(map[string]any) + if itemProps["summary"].(map[string]any)["minLength"] != 1 { t.Fatal("empty summaries permitted by schema") } - if repaired.Description() != "operator tool" || - props["summary"].(map[string]any)["description"] != "operator summary" || - props["confidence"].(map[string]any)["default"] != "inferred" || props["custom_field"] != nil { - t.Fatal("operator annotations lost") + if repaired.Description() != "operator tool" || facts["description"] != "operator facts" || + itemProps["summary"].(map[string]any)["description"] != "operator item summary" || + itemProps["confidence"].(map[string]any)["default"] != "observed" || itemProps["custom"] != nil { + t.Fatalf("item annotations lost or stray property kept: %v", itemProps) } if _, ok := old["properties"].(map[string]any)["tested_asset_ids"]; ok { t.Fatal("shared old schema mutated") @@ -94,9 +108,18 @@ func TestRecordFactStoredContractRepair(t *testing.T) { if err := json.Unmarshal(first, &roundTrip); err != nil { t.Fatal(err) } - for _, input := range []string{`{"summary":"single"}`, `{"facts":[{"summary":"batch"}]}`} { - if err := actool.ValidateInput(roundTrip, json.RawMessage(input)); err != nil { - t.Fatalf("SDK rejects valid %s after catalog round trip: %v", input, err) + if err := actool.ValidateInput(roundTrip, json.RawMessage(`{"facts":[{"summary":"batch"}]}`)); err != nil { + t.Fatalf("SDK rejects the published shape after catalog round trip: %v", err) + } + // The legacy single shape no longer passes the published schema; the + // handler validates its own input instead, also behind the catalog wrapper. + if err := actool.ValidateInput(roundTrip, json.RawMessage(`{"summary":"single"}`)); err == nil { + t.Fatal("published schema must require facts") + } + for _, tool := range []actool.CoreTool{base, repaired} { + r, ok := tool.(interface{ AcceptsRawInput() bool }) + if !ok || !r.AcceptsRawInput() { + t.Fatalf("%T must opt out of the harness schema check", tool) } } } @@ -140,7 +163,7 @@ func TestRecordFactStrictFields(t *testing.T) { } func TestRecordFactSchemaMatchesStrings(t *testing.T) { - props := recordFactSchema()["properties"].(map[string]any) + props := factItemSchema()["properties"].(map[string]any) summary := regexp.MustCompile(props["summary"].(map[string]any)["pattern"].(string)) for _, value := range []string{"", " ", "\t\r\n", "\u00a0\u0085", "\u2003\u3000", "conclusion", " observed "} { if got, want := summary.MatchString(value), strings.TrimSpace(value) != ""; got != want { @@ -155,7 +178,7 @@ func TestRecordFactSchemaMatchesStrings(t *testing.T) { } } for _, input := range []string{ - `{"summary":"single","asset_ids":["1",2]}`, + `{"facts":[{"summary":"batch","asset_ids":["1",2]}]}`, `{"facts":[{"summary":"batch","tested_asset_ids":["1",2]}]}`, } { if err := actool.ValidateInput(recordFactSchema(), json.RawMessage(input)); err != nil { @@ -163,3 +186,208 @@ func TestRecordFactSchemaMatchesStrings(t *testing.T) { } } } + +// A model that fills every declared property of the earlier schema sends the +// single fact inside facts[] and a copy (or a placeholder) of every fact field +// beside it. Such a call is refused by name before any write, whether the +// copies are identical, empty or dummies, and the message names the one shape +// that is accepted. The published schema no longer declares those fields. +func TestRecordFactRefusesTopLevelCopiesBesideFacts(t *testing.T) { + store := guardTaskStore(t) + intentID := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "fingerprint the service"}, "running") + tools := NewToolSet(store, "worker") + before, err := store.ListByKind(db.KindFact, 100) + if err != nil { + t.Fatal(err) + } + id := strconv.FormatInt(intentID, 10) + fact := `{"summary":"TLS 1.0 is offered on port 8443","detail":"the handshake succeeded with TLSv1.0","evidence":"openssl s_client -tls1 → Protocol: TLSv1","confidence":"observed","source":"tool_output","intent_id":` + id + `,"asset_ids":[]}` + for _, tc := range []struct{ call, want string }{ + // an exact copy of every field beside facts[] + {`{"summary":"TLS 1.0 is offered on port 8443","detail":"the handshake succeeded with TLSv1.0","evidence":"openssl s_client -tls1 → Protocol: TLSv1","confidence":"observed","source":"tool_output","intent_id":` + id + `,"asset_ids":[],"facts":[` + fact + `]}`, `"asset_ids", "confidence", "detail", "evidence", "source", "summary"`}, + // empty placeholders only + {`{"summary":"","detail":"","evidence":"","confidence":"","source":"","asset_ids":[],"tested_asset_ids":[],"intent_id":` + id + `,"facts":[` + fact + `]}`, `"asset_ids", "confidence", "detail", "evidence", "source", "summary", "tested_asset_ids"`}, + // dummy values + {`{"summary":"x","facts":[` + fact + `]}`, `"summary"`}, + {`{"summary":null,"facts":[` + fact + `]}`, `"summary"`}, + } { + w := tools.Writes().Facts + r := callGuardTool(t, tools.recordFact(), tc.call) + if !r.IsError || !strings.Contains(r.Flatten(), tc.want) || !strings.Contains(r.Flatten(), `record_fact takes only {"facts":[...]}`) || tools.Writes().Facts != w { + t.Errorf("%s: want refusal naming %s, got %s (writes %d→%d)", tc.call, tc.want, r.Flatten(), w, tools.Writes().Facts) + } + } + if n, err := store.ListByKind(db.KindFact, 100); err != nil || len(n) != len(before) { + t.Fatalf("refused calls wrote facts: %d → %d %v", len(before), len(n), err) + } + // The same fact in the published shape is written once and linked. + res := callGuardTool(t, tools.recordFact(), `{"facts":[`+fact+`]}`) + if res.IsError || tools.Writes().Facts != 1 { + t.Fatalf("published shape refused: %s", res.Flatten()) + } + var out struct { + IDs []int64 `json:"ids"` + } + if err := json.Unmarshal([]byte(res.Flatten()), &out); err != nil || len(out.IDs) != 1 || out.IDs[0] <= 0 { + t.Fatalf("batch result must carry exactly one id: %s", res.Flatten()) + } + if yielded, err := store.FactsYielded(intentID); err != nil || len(yielded) != 1 || yielded[0] != out.IDs[0] { + t.Fatalf("fact not linked to the intent: %v %v", yielded, err) + } + node, err := store.GetNode(out.IDs[0]) + if err != nil || node == nil { + t.Fatal(err) + } + var payload map[string]any + _ = json.Unmarshal(node.Payload, &payload) + if payload["summary"] != "TLS 1.0 is offered on port 8443" || payload["source"] != "tool_output" || payload["confidence"] != "observed" { + t.Fatalf("fact payload: %s", node.Payload) + } +} + +// Conflicting copies, placeholders, unknown fields, bad asset ids, missing +// summaries and empty batches are refused, and nothing is written by a +// refused call. +func TestRecordFactBatchTopLevelConflicts(t *testing.T) { + store := guardTaskStore(t) + tools := NewToolSet(store, "worker") + before, err := store.ListByKind(db.KindFact, 100) // the task's origin fact + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ call, want string }{ + // dummy summaries a model may send after a refusal + {`{"summary":"x","facts":[{"summary":"real conclusion"}]}`, `"summary"`}, + {`{"summary":"placeholder","detail":"test","evidence":"test","facts":[{"summary":"real","detail":"real detail","evidence":"real evidence"}]}`, `"detail", "evidence", "summary"`}, + {`{"summary":"contradictory","facts":[{"summary":"different"}]}`, `"summary"`}, + {`{"detail":"invented","facts":[{"summary":"legitimate"}]}`, `"detail"`}, + {`{"confidence":"observed","facts":[{"summary":"fact says inferred","confidence":"inferred"}]}`, `"confidence"`}, + {`{"source":"inference","facts":[{"summary":"fact says tool_output","source":"tool_output"}]}`, `"source"`}, + {`{"asset_ids":[1],"facts":[{"summary":"fact has other assets","asset_ids":[2]}]}`, `"asset_ids"`}, + {`{"summary":"first","facts":[{"summary":"first"},{"summary":"second"}]}`, `(2 item(s))`}, + {`{"summary":"x","typo":1,"facts":[{"summary":"unknown field"}]}`, `"typo"`}, + {`{"facts":[{"summary":"unknown nested","typo":1}]}`, `"typo"`}, + {`{"facts":[{"summary":"bad asset id","asset_ids":["abc"]}]}`, `asset`}, + {`{"summary":"must not inherit","facts":[{"detail":"missing summary"}]}`, `"summary"`}, + {`{"summary":"x","facts":[]}`, `non-empty`}, + {`{"facts":[]}`, `non-empty`}, + {`{"summary":"","facts":[]}`, `non-empty`}, + } { + w := tools.Writes().Facts + r := callGuardTool(t, tools.recordFact(), tc.call) + if !r.IsError || !strings.Contains(r.Flatten(), tc.want) || tools.Writes().Facts != w { + t.Errorf("%s: want refusal mentioning %s, got %s (writes %d→%d)", tc.call, tc.want, r.Flatten(), w, tools.Writes().Facts) + } + } + if n, err := store.ListByKind(db.KindFact, 100); err != nil || len(n) != len(before) { + t.Fatalf("refused calls wrote facts: %d → %d %v", len(before), len(n), err) + } +} + +// The published schema holds facts[] and nothing else, and the description +// names that one shape; the earlier "two shapes" wording is gone. +func TestRecordFactSchemaIsFactsOnly(t *testing.T) { + schema := recordFactSchema() + props := schema["properties"].(map[string]any) + if len(props) != 1 || props["facts"] == nil { + t.Fatalf("top-level properties: %v", props) + } + if req, _ := schema["required"].([]string); len(req) != 1 || req[0] != "facts" || schema["additionalProperties"] != false { + t.Fatalf("facts must be required and the only property: %v %v", schema["required"], schema["additionalProperties"]) + } + facts := props["facts"].(map[string]any) + if d, _ := facts["description"].(string); !strings.Contains(d, "never send summary") { + t.Errorf("facts description must forbid top-level copies: %q", d) + } + item := facts["items"].(map[string]any)["properties"].(map[string]any) + for _, name := range []string{"summary", "intent_id", "detail", "evidence", "confidence", "source", "asset_ids", "tested_asset_ids"} { + if item[name] == nil { + t.Errorf("facts[] item lacks %s", name) + } + } + for name, d := range map[string]string{ + "summary": item["summary"].(map[string]any)["description"].(string), + "intent_id": item["intent_id"].(map[string]any)["description"].(string), + } { + if strings.Contains(d, "top level") || strings.Contains(d, "top-level") { + t.Errorf("facts[].%s still refers to a top level: %q", name, d) + } + } + desc := NewToolSet(nil, "").recordFact().Description() + if !strings.HasSuffix(desc, recordFactContractNote) || strings.Contains(desc, "top-level intent_id") { + t.Fatalf("record_fact description must describe the facts-only contract: %q", desc) + } + // The seed the catalog stores is the same contract. + for _, seed := range BuiltinToolSeeds() { + if seed.Key != "record_fact" { + continue + } + if seed.Desc != desc { + t.Fatal("seed description differs from the tool") + } + if p := seed.Schema["properties"].(map[string]any); len(p) != 1 || p["facts"] == nil { + t.Fatalf("seed schema is not facts-only: %v", p) + } + } +} + +// A facts[] item without intent_id is linked to the intent the worker runs +// (SetOwnerNode); a legacy top-level intent_id still wins; a planner ToolSet +// (owner = the task root, not an intent) writes an unlinked fact. +func TestRecordFactDefaultsToRunningIntent(t *testing.T) { + store := guardTaskStore(t) + running := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "running intent"}, "running") + other := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "other intent"}, "open") + tools := NewToolSet(store, "worker") + tools.SetOwnerNode(running) + ids := func(res actool.Result) []int64 { + t.Helper() + var out struct { + IDs []int64 `json:"ids"` + } + if err := json.Unmarshal([]byte(res.Flatten()), &out); err != nil || len(out.IDs) != 1 || out.IDs[0] <= 0 { + t.Fatalf("batch result: %s", res.Flatten()) + } + return out.IDs + } + res := callGuardTool(t, tools.recordFact(), `{"facts":[{"summary":"linked to the running intent","confidence":"observed","source":"tool_output"}]}`) + if res.IsError { + t.Fatal(res.Flatten()) + } + first := ids(res)[0] + if yielded, err := store.FactsYielded(running); err != nil || len(yielded) != 1 || yielded[0] != first { + t.Fatalf("fact %d not linked to the running intent %d: %v %v", first, running, yielded, err) + } + res = callGuardTool(t, tools.recordFact(), `{"intent_id":`+strconv.FormatInt(other, 10)+`,"facts":[{"summary":"legacy top-level intent"}]}`) + if res.IsError { + t.Fatal(res.Flatten()) + } + second := ids(res)[0] + if yielded, err := store.FactsYielded(other); err != nil || len(yielded) != 1 || yielded[0] != second { + t.Fatalf("legacy top-level intent_id must win: %v %v", yielded, err) + } + res = callGuardTool(t, tools.recordFact(), `{"facts":[{"summary":"item intent","intent_id":`+strconv.FormatInt(other, 10)+`}]}`) + if res.IsError { + t.Fatal(res.Flatten()) + } + third := ids(res)[0] + if yielded, _ := store.FactsYielded(other); len(yielded) != 2 || yielded[1] != third && yielded[0] != third { + t.Fatalf("item intent_id must win: %v", yielded) + } + if yielded, _ := store.FactsYielded(running); len(yielded) != 1 { + t.Fatalf("running intent gained facts it did not produce: %v", yielded) + } + planner := NewToolSet(store, "planner") + origin, err := store.ListByKind(db.KindFact, 1) + if err != nil || len(origin) == 0 { + t.Fatalf("origin fact: %v", err) + } + planner.SetOwnerNode(origin[0].ID) + res = callGuardTool(t, planner.recordFact(), `{"facts":[{"summary":"planner fact without intent"}]}`) + if res.IsError || planner.Writes().Facts != 1 { + t.Fatalf("planner write refused: %s", res.Flatten()) + } + if yielded, _ := store.FactsYielded(running); len(yielded) != 1 { + t.Fatalf("planner fact must not be linked to a worker intent: %v", yielded) + } +} diff --git a/agent/record_fact_shapes_test.go b/agent/record_fact_shapes_test.go new file mode 100644 index 000000000..98835a3d3 --- /dev/null +++ b/agent/record_fact_shapes_test.go @@ -0,0 +1,128 @@ +package agent + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "testing" + + "github.com/Autumn-27/artex/db" +) + +// Synthetic replay of the record_fact call shapes a model produces, replayed +// against the handler in the package test database. The shapes mirror what +// was observed in production before the facts-only contract (a model filling +// every declared top-level property beside facts[]) without using recorded +// data. Checked per call: the verdict, that an accepted call writes exactly one +// node per facts item linked to the intent and anchored to the assets, and +// that a refused call writes nothing and names the top-level fields. +func TestRecordFactShapesReplay(t *testing.T) { + d := testDB(t) + t.Cleanup(func() { _ = d.Close() }) + task, err := d.CreateTaskWithOptions("record_fact shapes", "replay synthetic record_fact call shapes", db.TaskCreateOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = d.DeleteTask(task.ID) }) + store := d.Exploration(task.ExplorationID) + intentID := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "enumerate the service"}, "running") + var assets []int64 + for i := 1; i <= 2; i++ { + id, err := d.Assets().UpsertIP(db.UpsertIPReq{IP: fmt.Sprintf("203.0.113.%d", i), TaskID: task.ID}) + if err != nil { + t.Fatal(err) + } + assets = append(assets, id) + } + t.Cleanup(func() { _, _ = d.Assets().DeleteByIDs(assets) }) + tools := NewToolSet(store, "worker") + tools.SetOwnerNode(intentID) + tool := tools.recordFact() + + iid := strconv.FormatInt(intentID, 10) + a1, a2 := strconv.FormatInt(assets[0], 10), strconv.FormatInt(assets[1], 10) + item := func(n int) string { + return fmt.Sprintf(`{"summary":"conclusion %d","detail":"detail %d","evidence":"cmd %d → line","confidence":"observed","source":"tool_output","asset_ids":[%s]}`, n, n, n, a1) + } + type shape struct { + name string + call string + accept bool + items int + fields string // named in the refusal + } + shapes := []shape{ + {"published: one item", `{"facts":[` + item(1) + `]}`, true, 1, ""}, + {"published: two items", `{"facts":[` + item(1) + `,` + item(2) + `]}`, true, 2, ""}, + {"published: item with its own intent_id and alias", `{"facts":[{"summary":"alias","tested_asset_ids":["` + a2 + `"],"intent_id":` + iid + `}]}`, true, 1, ""}, + {"legacy batch: top-level intent_id", `{"intent_id":` + iid + `,"facts":[` + item(3) + `]}`, true, 1, ""}, + {"legacy single", `{"summary":"single fact","confidence":"inferred","asset_ids":[` + a2 + `]}`, true, 1, ""}, + {"copies of every field beside facts", `{"summary":"conclusion 1","detail":"detail 1","evidence":"cmd 1 → line","confidence":"observed","source":"tool_output","asset_ids":[` + a1 + `],"facts":[` + item(1) + `]}`, false, 1, `"asset_ids", "confidence", "detail", "evidence", "source", "summary"`}, + {"empty placeholders beside facts", `{"summary":"","detail":"","asset_ids":[],"tested_asset_ids":[],"facts":[` + item(1) + `]}`, false, 1, `"asset_ids", "detail", "summary", "tested_asset_ids"`}, + {"dummy summary beside facts", `{"summary":"x","facts":[` + item(1) + `]}`, false, 1, `"summary"`}, + {"abbreviated copy beside facts", `{"summary":"short","detail":"short detail","facts":[` + item(1) + `]}`, false, 1, `"detail", "summary"`}, + {"conflicting source beside facts", `{"source":"inference","facts":[` + item(1) + `]}`, false, 1, `"source"`}, + {"top-level summary beside two items", `{"summary":"conclusion 1","facts":[` + item(1) + `,` + item(2) + `]}`, false, 2, `"summary"`}, + } + for _, s := range shapes { + before := tools.Writes().Facts + linked, _ := store.FactsYielded(intentID) + res := callGuardTool(t, tool, s.call) + written := tools.Writes().Facts - before + if !res.IsError != s.accept { + t.Errorf("%s: accepted=%v want %v: %s", s.name, !res.IsError, s.accept, res.Flatten()) + continue + } + if !s.accept { + if written != 0 || !strings.Contains(res.Flatten(), "top-level field(s) "+s.fields) || !strings.Contains(res.Flatten(), "no facts were written") { + t.Errorf("%s: refusal must write nothing and name %s: %s (writes %d)", s.name, s.fields, res.Flatten(), written) + } + continue + } + if written != s.items { + t.Errorf("%s: wrote %d facts for %d items", s.name, written, s.items) + } + after, _ := store.FactsYielded(intentID) + if len(after) != len(linked)+s.items { + t.Errorf("%s: intent links %d → %d for %d items", s.name, len(linked), len(after), s.items) + } + ids := after[len(linked):] + if strings.HasPrefix(s.call, `{"facts"`) || strings.HasPrefix(s.call, `{"intent_id"`) { + var out struct { + IDs []int64 `json:"ids"` + } + if err := json.Unmarshal([]byte(res.Flatten()), &out); err != nil || len(out.IDs) != s.items { + t.Errorf("%s: ids: %s", s.name, res.Flatten()) + } + } + for _, id := range ids { + node, err := store.GetNode(id) + if err != nil || node == nil || node.Kind != db.KindFact { + t.Fatalf("%s: fact %d: %v", s.name, id, err) + } + var payload map[string]any + _ = json.Unmarshal(node.Payload, &payload) + if sum, _ := payload["summary"].(string); sum == "" || sum == "x" { + t.Errorf("%s: stored summary %q", s.name, sum) + } + if strings.Contains(s.call, "asset_ids") { + anchored := false + for _, asset := range assets { + refs, err := store.AssetRefs(asset) + if err != nil { + t.Fatal(err) + } + for _, r := range refs { + if r.ID == id { + anchored = true + } + } + } + if !anchored { + t.Errorf("%s: fact %d not anchored to an asset", s.name, id) + } + } + } + } +} diff --git a/agent/toolcatalog.go b/agent/toolcatalog.go index 5811203b5..6e85c7eb4 100644 --- a/agent/toolcatalog.go +++ b/agent/toolcatalog.go @@ -180,6 +180,14 @@ type overriddenTool struct { func (o *overriddenTool) Description() string { return o.desc } func (o *overriddenTool) InputSchema() map[string]any { return o.schema } +// AcceptsRawInput forwards the wrapped tool's opt-out of the harness schema +// check (record_fact validates its own input); the embedded interface alone +// would hide it from the harness's type assertion. +func (o *overriddenTool) AcceptsRawInput() bool { + r, ok := o.CoreTool.(interface{ AcceptsRawInput() bool }) + return ok && r.AcceptsRawInput() +} + func (o *overriddenTool) Call(ctx context.Context, in json.RawMessage, tc *actool.ToolContext) (actool.Result, error) { return o.CoreTool.Call(ctx, injectDefaults(in, o.schema), tc) } diff --git a/agent/tools.go b/agent/tools.go index a34f0e1dd..939871cce 100644 --- a/agent/tools.go +++ b/agent/tools.go @@ -394,8 +394,21 @@ func readTool(name, desc string, schema map[string]any, run func(context.Context } func writeTool(name, desc string, schema map[string]any, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool { + return buildWriteTool(name, desc, schema, false, run) +} + +// writeRawTool is writeTool for a handler that validates its own input: the +// harness skips its schema check (Spec.RawInput), so the published schema can +// be stricter (required properties) than the legacy shapes the handler still +// accepts, and refusals carry the handler's own message instead of the +// SDK's generic one. +func writeRawTool(name, desc string, schema map[string]any, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool { + return buildWriteTool(name, desc, schema, true, run) +} + +func buildWriteTool(name, desc string, schema map[string]any, raw bool, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool { return actool.Build(actool.Spec{ - Name: name, Description: desc, Schema: schema, + Name: name, Description: desc, Schema: schema, RawInput: raw, Permissions: func(context.Context, json.RawMessage, acperm.Context) acperm.Decision { return acperm.Allowed() }, Run: func(ctx context.Context, in json.RawMessage, _ *actool.ToolContext) (actool.Result, error) { return run(ctx, in) @@ -419,6 +432,12 @@ func (t *ToolSet) writeExpTool(name, desc string, schema map[string]any, run fun return writeTool(name, desc, schema, t.needExploration(name, run)) } +// writeExpToolRaw is writeExpTool with the harness schema check skipped (see +// writeRawTool); the handler validates the input itself. +func (t *ToolSet) writeExpToolRaw(name, desc string, schema map[string]any, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool { + return writeRawTool(name, desc, schema, t.needExploration(name, run)) +} + // needExploration wraps a handler so it only runs with an exploration store. // Tools that degrade more usefully than "unavailable" (report_finding points at // add_task_hint, set_goals/set_constraints at the task itself) keep their own @@ -1588,15 +1607,30 @@ func (t *ToolSet) recordOneFact(it factItem, defaultIntent int64) (int64, error) return id, nil } +// runningIntent is the intent this ToolSet works for (the worker's claimed +// intent, set by SetOwnerNode), or 0 when the owner node is not an intent +// (planner: the task root) or unknown. +func (t *ToolSet) runningIntent() int64 { + if t.ts == nil || t.ownerNode <= 0 { + return 0 + } + node, err := t.ts.GetNode(t.ownerNode) + if err != nil || node == nil || node.Kind != db.KindIntent { + return 0 + } + return node.ID +} + func (t *ToolSet) recordFact() actool.CoreTool { - return t.writeExpTool("record_fact", "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n"+ + return t.writeExpToolRaw("record_fact", "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n"+ "⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n"+ - "★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n"+ + "★facts is the only parameter: facts:[{...}] with one item per [mutually distinct] conclusion, usually exactly one item. Returns an ids array, same length and order as facts. An item without intent_id is linked to the intent you are running.\n"+ "⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n"+ " · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n"+ " · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n"+ " · source = where the content came from: target_response (content the target itself returned: page bodies, headers, error messages, banners, even when a tool fetched it) | tool_output (a tool's own result about the target: port states, scanner verdicts, status codes, timings) | third_party_intel (OSINT, CVE databases, public reports) | operator_input (given by the human operator) | inference (your own reasoning; the default when omitted). Any other value is refused.\n"+ - " · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen.", + " · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen."+ + recordFactContractNote, recordFactSchema(), func(_ context.Context, in json.RawMessage) (actool.Result, error) { a, batch, err := decodeRecordFactInput(in) @@ -1607,7 +1641,12 @@ func (t *ToolSet) recordFact() actool.CoreTool { if !batch { items = []factItem{a.factItem} } - defaultIntent := pid(a.factItem.IntentID) // top-level intent_id = batch default + // Default intent of the items: a legacy top-level intent_id, else the + // intent this worker is running (the published schema has no top level). + defaultIntent := pid(a.factItem.IntentID) + if defaultIntent <= 0 { + defaultIntent = t.runningIntent() + } ids := make([]int64, len(items)) errs := map[string]string{} diff --git a/server/fact_input_schema.go b/server/fact_input_schema.go index 22ffed682..7ca01c1be 100644 --- a/server/fact_input_schema.go +++ b/server/fact_input_schema.go @@ -9,6 +9,85 @@ import ( // SeedTool only inserts new rows. Repair existing built-ins too, while keeping // descriptions, bindings, enabled/deferred flags and parameter annotations. + +// refreshRecordFactContract brings an already-seeded record_fact row to the +// facts-only contract (one-time, settings flag). SeedTool is first-insert-only +// and ToolResolve serves the stored text, so without this an old database keeps +// describing the earlier contract (fact fields at the top level beside facts[]) +// to the model. Only text equal to the previous code default is replaced: +// - the tool description; +// - the facts property and item descriptions that referred to the top level. +// +// Operator edits stay and are named in the log. The schema structure itself is +// repaired by seedFactInputSchema on every start (RepairRecordFactSchema). +func (s *Server) refreshRecordFactContract() { + const flag = "tool_schema_refresh_record_fact_facts_only" + if v, _, _ := s.m.pg.GetSetting(flag); v == "true" { + return + } + defer func() { _ = s.m.pg.SetSetting(flag, "true") }() + var seed *agent.ToolSeed + for _, sd := range agent.BuiltinToolSeeds() { + if sd.Key == "record_fact" { + seed = &sd + break + } + } + row, err := s.m.pg.GetTool("record_fact") + if err != nil || row == nil || seed == nil || !row.System || row.Kind != "builtin" { + return + } + legacyTool, legacyFacts, legacyItems := agent.RecordFactLegacyDescriptions() + desc := row.Description + switch { + case desc == seed.Desc: + case sliceHas(legacyTool, desc): + desc = seed.Desc + default: + log.Printf("[facts] record_fact description was edited by an operator and still describes the earlier contract; the facts-only schema is published anyway, review the text in the tool catalog") + } + var schema map[string]any + if err := json.Unmarshal(row.Schema, &schema); err != nil { + log.Printf("[facts] decode record_fact schema: %v", err) + return + } + repaired := agent.RepairRecordFactSchema(schema) + facts := repaired["properties"].(map[string]any)["facts"].(map[string]any) + canonFacts := seed.Schema["properties"].(map[string]any)["facts"].(map[string]any) + if facts["description"] == legacyFacts { + facts["description"] = canonFacts["description"] + } + items := facts["items"].(map[string]any)["properties"].(map[string]any) + canonItems := canonFacts["items"].(map[string]any)["properties"].(map[string]any) + for name, old := range legacyItems { + prop, _ := items[name].(map[string]any) + want, _ := canonItems[name].(map[string]any) + if prop != nil && want != nil && prop["description"] == old { + prop["description"] = want["description"] + } + } + raw, err := json.Marshal(repaired) + if err != nil { + return + } + if _, err := s.m.pg.Exec(`UPDATE tools SET description=$1, schema=$2::jsonb, updated_at=now() +WHERE key='record_fact' AND system AND kind='builtin' + AND (description IS DISTINCT FROM $1 OR schema IS DISTINCT FROM $2::jsonb)`, desc, string(raw)); err != nil { + log.Printf("[facts] refresh record_fact contract: %v", err) + return + } + log.Printf("[facts] record_fact now publishes the facts-only contract (one-time)") +} + +func sliceHas(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + func (s *Server) seedFactInputSchema() { row, err := s.m.pg.GetTool("record_fact") if err != nil { diff --git a/server/fact_input_schema_test.go b/server/fact_input_schema_test.go index 4114633a6..9cf9df34a 100644 --- a/server/fact_input_schema_test.go +++ b/server/fact_input_schema_test.go @@ -3,6 +3,7 @@ package server import ( "context" "encoding/json" + "strings" "testing" "time" @@ -42,10 +43,16 @@ func TestRecordFactInputSchemaUpgrade(t *testing.T) { if err := pg.UpdateTool("record_fact", "operator tool", raw, bindings, old.Enabled); err != nil { t.Fatal(err) } + // Facts-only contract: the top level holds facts and + // nothing else; the earlier top-level fact fields and their operator + // annotations are dropped (the handler still accepts the legacy shapes), + // item annotations survive. check := func(schema map[string]any) { t.Helper() - if _, ok := schema["required"]; ok { - t.Fatal("top-level summary still required for batches") + if req, _ := schema["required"].([]any); len(req) != 1 || req[0] != "facts" { + if reqS, _ := schema["required"].([]string); len(reqS) != 1 || reqS[0] != "facts" { + t.Fatalf("facts must be the only required property: %v", schema["required"]) + } } if _, ok := schema["anyOf"]; ok { t.Fatal("root anyOf still present") @@ -54,14 +61,13 @@ func TestRecordFactInputSchemaUpgrade(t *testing.T) { t.Fatal("unknown fields allowed") } props := schema["properties"].(map[string]any) + if len(props) != 1 || props["facts"] == nil { + t.Fatalf("top level must hold facts only: %v", props) + } items := props["facts"].(map[string]any)["items"].(map[string]any) itemProps := items["properties"].(map[string]any) - if props["tested_asset_ids"] == nil || itemProps["tested_asset_ids"] == nil || itemProps["summary"] == nil { - t.Fatal("repaired schema missing alias or batch summary") - } - if props["summary"].(map[string]any)["description"] != "operator summary" || - props["confidence"].(map[string]any)["default"] != "inferred" || props["custom"] != nil { - t.Fatal("operator annotations lost") + if itemProps["tested_asset_ids"] == nil || itemProps["summary"] == nil || itemProps["source"] == nil { + t.Fatal("repaired schema missing alias, batch summary or source") } if err := actool.ValidateInput(schema, json.RawMessage(`{"facts":[{"summary":"complete batch item"}]}`)); err != nil { t.Fatal("valid batch rejected", err) @@ -104,3 +110,134 @@ func TestRecordFactInputSchemaUpgrade(t *testing.T) { t.Fatal("second migration changed the row") } } + +// recordFactLegacySchema is the stored record_fact schema of the earlier +// contract: fact fields at the top level beside facts[], with the texts that +// referred to the top level, plus an operator annotation on an item field. +func recordFactLegacySchema(t *testing.T) (string, map[string]string) { + t.Helper() + _, legacyFacts, legacyItems := agent.RecordFactLegacyDescriptions() + item := map[string]any{"type": "object", "required": []string{"summary"}, "additionalProperties": false, "properties": map[string]any{ + "summary": map[string]any{"type": "string", "minLength": 1, "description": legacyItems["summary"]}, + "intent_id": map[string]any{"type": "integer", "description": legacyItems["intent_id"]}, + "detail": map[string]any{"type": "string", "description": "Details of this fact, including all related observations."}, + "confidence": map[string]any{"type": "string", "description": "operator confidence note", "default": "inferred"}, + "source": map[string]any{"type": "string", "enum": []string{"target_response", "tool_output", "third_party_intel", "operator_input", "inference"}, "description": "where the content came from"}, + }} + schema := map[string]any{"type": "object", "additionalProperties": false, "properties": map[string]any{ + "summary": map[string]any{"type": "string", "minLength": 1, "description": legacyItems["summary"]}, + "intent_id": map[string]any{"type": "integer", "description": legacyItems["intent_id"]}, + "detail": map[string]any{"type": "string", "description": "Details of this fact, including all related observations."}, + "source": map[string]any{"type": "string", "enum": []string{"inference"}}, + "facts": map[string]any{"type": "array", "minItems": 1, "description": legacyFacts, "items": item}, + }} + raw, _ := json.Marshal(schema) + return string(raw), legacyItems +} + +func recordFactSeed(t *testing.T) agent.ToolSeed { + t.Helper() + for _, sd := range agent.BuiltinToolSeeds() { + if sd.Key == "record_fact" { + return sd + } + } + t.Fatal("record_fact seed missing") + return agent.ToolSeed{} +} + +// A database that still stores the earlier record_fact contract (text and +// schema) is brought to the facts-only contract once: description replaced +// when it is the previous default, facts/item descriptions replaced when they +// are the earlier texts, operator annotations kept, top-level fact fields +// gone, source enum present on the items. A second run changes nothing, and +// an operator-edited description is left alone. +func TestRefreshRecordFactContract(t *testing.T) { + skipWithoutDSN(t) + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Skipf("postgres unavailable (%v)", err) + } + defer m.Close() + s := &Server{m: m} + seed := recordFactSeed(t) + seedSchema, _ := json.Marshal(seed.Schema) + seedAgents, _ := json.Marshal(seed.Agents) + restore := func() { + _ = m.pg.UpsertToolForce("record_fact", seed.Desc, seedSchema, seedAgents) + _, _ = m.pg.Exec(`DELETE FROM settings WHERE key='tool_schema_refresh_record_fact_facts_only'`) + } + defer restore() + + legacyTool, _, legacyItems := agent.RecordFactLegacyDescriptions() + legacySchema, _ := recordFactLegacySchema(t) + run := func(desc string) { + t.Helper() + if err := m.pg.UpsertToolForce("record_fact", desc, json.RawMessage(legacySchema), seedAgents); err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec(`DELETE FROM settings WHERE key='tool_schema_refresh_record_fact_facts_only'`); err != nil { + t.Fatal(err) + } + s.seedFactInputSchema() + s.seedFactSourceSchema() + s.refreshRecordFactContract() + } + for _, desc := range legacyTool { + run(desc) + row, err := m.pg.GetTool("record_fact") + if err != nil || row == nil { + t.Fatalf("row: %v", err) + } + if row.Description != seed.Desc { + t.Fatalf("description not refreshed from a known default: %q", row.Description[:60]) + } + var schema map[string]any + if err := json.Unmarshal(row.Schema, &schema); err != nil { + t.Fatal(err) + } + props := schema["properties"].(map[string]any) + if len(props) != 1 || props["facts"] == nil { + t.Fatalf("top level still holds fact fields: %v", props) + } + facts := props["facts"].(map[string]any) + items := facts["items"].(map[string]any)["properties"].(map[string]any) + canonItems := seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any) + if facts["description"] != seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)["description"] { + t.Fatalf("facts description not refreshed: %v", facts["description"]) + } + for name := range legacyItems { + if items[name].(map[string]any)["description"] != canonItems[name].(map[string]any)["description"] { + t.Fatalf("item %s description not refreshed: %v", name, items[name]) + } + } + if items["confidence"].(map[string]any)["description"] != "operator confidence note" || items["confidence"].(map[string]any)["default"] != "inferred" { + t.Fatalf("operator annotation lost: %v", items["confidence"]) + } + if enum, _ := items["source"].(map[string]any)["enum"].([]any); len(enum) != 5 { + t.Fatalf("source enum missing on the items: %v", items["source"]) + } + if v, _, _ := m.pg.GetSetting("tool_schema_refresh_record_fact_facts_only"); v != "true" { + t.Fatal("flag not set") + } + // Second start: nothing changes. + var updated1, updated2 string + _ = m.pg.QueryRow(`SELECT updated_at::text FROM tools WHERE key='record_fact'`).Scan(&updated1) + s.seedFactInputSchema() + s.seedFactSourceSchema() + s.refreshRecordFactContract() + _ = m.pg.QueryRow(`SELECT updated_at::text FROM tools WHERE key='record_fact'`).Scan(&updated2) + if updated1 != updated2 { + t.Fatal("second start rewrote the row") + } + } + // An operator-edited description is kept; the schema is still repaired. + run("my own record_fact text") + row, _ := m.pg.GetTool("record_fact") + if row == nil || row.Description != "my own record_fact text" { + t.Fatalf("operator description replaced: %v", row) + } + if !strings.Contains(string(row.Schema), `"facts"`) || strings.Contains(string(row.Schema), legacyItems["intent_id"]) { + t.Fatalf("schema not repaired beside an operator description: %s", row.Schema) + } +} diff --git a/server/fact_source_schema_test.go b/server/fact_source_schema_test.go index a087b4fc0..baa30931a 100644 --- a/server/fact_source_schema_test.go +++ b/server/fact_source_schema_test.go @@ -35,12 +35,13 @@ func TestRecordFactSourceSchemaUpgrade(t *testing.T) { t.Cleanup(func() { pg.UpdateTool(old.Key, old.Description, old.Schema, bindings, old.Enabled) }) // Simulate a pre-provenance row customised by the operator: no source - // property, an edited description and an edited parameter description. + // property on the facts items, an edited description and an edited + // parameter description (facts-only contract: the items carry the fields). var schema map[string]any if err := json.Unmarshal(old.Schema, &schema); err != nil { t.Fatal(err) } - props := schema["properties"].(map[string]any) + props := schema["properties"].(map[string]any)["facts"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any) delete(props, "source") props["confidence"].(map[string]any)["description"] = "operator confidence text" custom, _ := json.Marshal(schema) @@ -72,16 +73,23 @@ func TestRecordFactSourceSchemaUpgrade(t *testing.T) { t.Fatal(err) } var got struct { - Properties map[string]map[string]any `json:"properties"` + Properties struct { + Facts struct { + Items struct { + Properties map[string]map[string]any `json:"properties"` + } `json:"items"` + } `json:"facts"` + } `json:"properties"` } if err := json.Unmarshal(tool.Schema, &got); err != nil { t.Fatal(err) } + itemProps := got.Properties.Facts.Items.Properties wantEnum := "[target_response tool_output third_party_intel operator_input inference]" - if src := got.Properties["source"]; src == nil || fmt.Sprint(src["enum"]) != wantEnum { + if src := itemProps["source"]; src == nil || fmt.Sprint(src["enum"]) != wantEnum { t.Fatalf("upgrade did not add the source enum: %s", tool.Schema) } - if tool.Description != "operator description" || got.Properties["confidence"]["description"] != "operator confidence text" || len(got.Properties) != len(props)+1 { + if tool.Description != "operator description" || itemProps["confidence"]["description"] != "operator confidence text" || len(itemProps) != len(props)+1 { t.Fatalf("operator configuration overwritten: %q %s", tool.Description, tool.Schema) } @@ -92,15 +100,18 @@ func TestRecordFactSourceSchemaUpgrade(t *testing.T) { } } -// resolvedRecordFactProps returns the properties of record_fact as ToolResolve -// serves it to the worker. +// resolvedRecordFactProps returns the facts[] item properties of record_fact as +// ToolResolve serves it to the worker. func resolvedRecordFactProps(t *testing.T) map[string]any { t.Helper() tools := agent.ToolResolve()(context.Background(), "worker", agent.NewToolSet(nil, "").WorkerTools()) for _, tl := range tools { if tl.Name() == "record_fact" { props, _ := tl.InputSchema()["properties"].(map[string]any) - return props + facts, _ := props["facts"].(map[string]any) + items, _ := facts["items"].(map[string]any) + itemProps, _ := items["properties"].(map[string]any) + return itemProps } } t.Fatal("worker lost record_fact") diff --git a/server/orchestration.go b/server/orchestration.go index bf88343e6..9b90deb75 100644 --- a/server/orchestration.go +++ b/server/orchestration.go @@ -573,6 +573,7 @@ func (s *Server) seedOrchestrationTools() { s.seedFindingEvidenceSchema() // optional counterevidence / confidence_rationale on report_finding, added once per DB s.seedFactInputSchema() // restore record_fact single/batch contracts s.seedFactSourceSchema() // source enum on record_fact, added once per DB + s.refreshRecordFactContract() // record_fact publishes the facts-only contract (one-time) s.seedTaskScopeDescription() // add_task_scope: "coverage scope, not enforced" replaces the authorization-boundary text // Note: pentest's default tool bindings need no migration — BuiltinToolSeeds already seeds // list_assets/insert_assets/report_finding/list_findings/list_companies together with @@ -602,11 +603,23 @@ func (s *Server) seedFactSourceSchema() { if err := json.Unmarshal(obj["properties"], &properties); err != nil { return } - if len(properties["source"]) == 0 { + // The source enum lives on the facts[] items (facts-only contract); the + // top level has no fact fields any more. + var facts map[string]json.RawMessage + var items map[string]json.RawMessage + var itemProps map[string]json.RawMessage + if err := json.Unmarshal(properties["facts"], &facts); err != nil { return } - _, err = s.m.pg.Exec(`UPDATE tools SET schema=jsonb_set(schema,ARRAY['properties','source'],$1::jsonb,true),updated_at=now() -WHERE key='record_fact' AND system AND NOT(COALESCE(schema->'properties','{}'::jsonb) ? 'source')`, string(properties["source"])) + if err := json.Unmarshal(facts["items"], &items); err != nil { + return + } + if err := json.Unmarshal(items["properties"], &itemProps); err != nil || len(itemProps["source"]) == 0 { + return + } + _, err = s.m.pg.Exec(`UPDATE tools SET schema=jsonb_set(schema,ARRAY['properties','facts','items','properties','source'],$1::jsonb,true),updated_at=now() +WHERE key='record_fact' AND system AND (schema->'properties'->'facts'->'items') ? 'properties' + AND NOT(COALESCE(schema->'properties'->'facts'->'items'->'properties','{}'::jsonb) ? 'source')`, string(itemProps["source"])) if err != nil { log.Printf("[facts] upgrade record_fact.source: %v", err) } diff --git a/server/tools_wire_test.go b/server/tools_wire_test.go index 89ebf2faf..7a168d8db 100644 --- a/server/tools_wire_test.go +++ b/server/tools_wire_test.go @@ -73,12 +73,19 @@ func TestWireTools(t *testing.T) { // Edit description + add a default to a scalar param, then confirm the resolved // worker tool reflects both. Restore from code default on cleanup. t.Cleanup(func() { _ = pg.UpsertToolForce(rf.Key, rf.Description, rf.Schema, mustJSON(rf.Agents)) }) + // Facts-only contract: the fact fields, and so the + // editable scalar annotations, live on the facts[] items. var schema map[string]any _ = json.Unmarshal(rf.Schema, &schema) - if props, ok := schema["properties"].(map[string]any); ok { - if conf, ok := props["confidence"].(map[string]any); ok { - conf["default"] = "inferred" - } + itemProps := func(sc map[string]any) map[string]any { + props, _ := sc["properties"].(map[string]any) + facts, _ := props["facts"].(map[string]any) + items, _ := facts["items"].(map[string]any) + ip, _ := items["properties"].(map[string]any) + return ip + } + if conf, ok := itemProps(schema)["confidence"].(map[string]any); ok { + conf["default"] = "inferred" } edited := mustJSON(schema) if err := pg.UpdateTool(rf.Key, "EDITED DESC", edited, mustJSON([]string{"worker"}), true); err != nil { @@ -96,9 +103,8 @@ func TestWireTools(t *testing.T) { // Default injection: omit confidence → handler input should carry the default. // We can't run the real handler (nil store), but InputSchema must show the default. sc := got.InputSchema() - props := sc["properties"].(map[string]any) - conf := props["confidence"].(map[string]any) - if conf["default"] != "inferred" { + conf, _ := itemProps(sc)["confidence"].(map[string]any) + if conf == nil || conf["default"] != "inferred" { t.Errorf("confidence.default = %v, want inferred", conf["default"]) } } From c5cb1afad7e4e3e814e04e0df7e6cf52b7d1be40 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Fri, 20 Nov 2026 18:11:16 +0100 Subject: [PATCH 2/6] fix(agent): keep the record_fact harness opt-out through tool wrappers record_fact opts out of the harness schema check (norma Spec.RawInput) and validates its own input. The harness finds the opt-out with a type assertion on the tool it is handed, and that tool is a wrapper: catalog tools are metered (meteredTool), and guardPanic wraps every tool last (guardedTool). Both embed CoreTool, which hides AcceptsRawInput, so the opt-out was lost and a malformed call was answered by the SDK ("invalid tool input: facts: expected array") instead of by the handler. guardedTool and meteredTool now forward AcceptsRawInput, as overriddenTool already does. Test: TestFix410AcceptsRawInputSurvivesWrappers covers each wrapper and record_fact through wireTools, AugmentTools and harness.Query. --- agent/assembly.go | 8 ++ server/record_fact_rawinput_test.go | 119 ++++++++++++++++++++++++++++ server/tool_usage.go | 8 ++ 3 files changed, 135 insertions(+) create mode 100644 server/record_fact_rawinput_test.go diff --git a/agent/assembly.go b/agent/assembly.go index c0885d94e..4a075e79d 100644 --- a/agent/assembly.go +++ b/agent/assembly.go @@ -101,6 +101,14 @@ func guardPanic(t actool.CoreTool) actool.CoreTool { return &guardedTool{CoreToo type guardedTool struct{ actool.CoreTool } +// AcceptsRawInput forwards the wrapped tool's opt-out of the harness schema +// check, as overriddenTool does: this wrapper is the one the harness sees, and +// the embedded interface alone would hide the opt-out from its type assertion. +func (g *guardedTool) AcceptsRawInput() bool { + r, ok := g.CoreTool.(interface{ AcceptsRawInput() bool }) + return ok && r.AcceptsRawInput() +} + func (g *guardedTool) Call(ctx context.Context, in json.RawMessage, tc *actool.ToolContext) (res actool.Result, err error) { defer func() { if r := recover(); r != nil { diff --git a/server/record_fact_rawinput_test.go b/server/record_fact_rawinput_test.go new file mode 100644 index 000000000..92a643abd --- /dev/null +++ b/server/record_fact_rawinput_test.go @@ -0,0 +1,119 @@ +package server + +import ( + "context" + "strings" + "testing" + + "github.com/Autumn-27/norma/harness" + "github.com/Autumn-27/norma/llm" + actool "github.com/Autumn-27/norma/tool" + + "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/db" +) + +// acceptsRawInput is the assertion norma's harness makes before its schema +// check: only a tool that reports AcceptsRawInput skips it. +func acceptsRawInput(t actool.CoreTool) bool { + r, ok := t.(interface{ AcceptsRawInput() bool }) + return ok && r.AcceptsRawInput() +} + +// record_fact opts out of the harness schema check and validates its own +// input. Every wrapper between the handler and the harness has to forward +// that opt-out: an embedded CoreTool alone hides it from the harness's type +// assertion, and the stricter published schema then rejects the calls the +// handler exists to answer. +func TestFix410AcceptsRawInputSurvivesWrappers(t *testing.T) { + oldAugment, oldResolve := agent.ToolAugment(), agent.ToolResolve() + t.Cleanup(func() { + agent.SetToolAugment(oldAugment) + agent.SetToolResolve(oldResolve) + }) + ctx := context.Background() + probe := func(raw bool) actool.CoreTool { + return actool.Build(actool.Spec{Name: "fix410_probe", Schema: map[string]any{"type": "object"}, RawInput: raw}) + } + + // AugmentTools without hooks applies guardPanic and nothing else. + t.Run("guardedTool", func(t *testing.T) { + agent.SetToolAugment(nil) + agent.SetToolResolve(nil) + for _, raw := range []bool{true, false} { + out, _, cleanup := agent.AugmentTools(ctx, "worker", []actool.CoreTool{probe(raw)}) + cleanup() + if len(out) != 1 { + t.Fatalf("AugmentTools returned %d tools, want 1", len(out)) + } + if got := acceptsRawInput(out[0]); got != raw { + t.Errorf("%T: AcceptsRawInput = %v, wrapped tool reports %v", out[0], got, raw) + } + } + }) + + t.Run("meteredTool", func(t *testing.T) { + for _, raw := range []bool{true, false} { + wrapped := meterTool(probe(raw), &recordingToolUsage{}, "fix410_probe", "worker", agent.RunInfo{}) + if got := acceptsRawInput(wrapped); got != raw { + t.Errorf("%T: AcceptsRawInput = %v, wrapped tool reports %v", wrapped, got, raw) + } + } + }) + + // The chain a worker's record_fact goes through on the server: the catalog + // decorator and the usage meter (ToolResolve, wireTools), then guardPanic + // (AugmentTools), then the harness itself. + t.Run("server chain", func(t *testing.T) { + dsn, _, err := db.DSN() + if err != nil { + db.SkipPGUnavailable(t, err) + } + pg, err := db.Open(dsn) + if err != nil { + db.SkipPGUnavailable(t, err) + } + defer pg.Close() + agent.SetToolAugment(nil) + wireTools(pg, nil) + task, err := pg.CreateTaskWithOptions("fix410", "record_fact through the wrapper chain", db.TaskCreateOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = pg.DeleteTask(task.ID) }) + ts := agent.NewToolSet(pg.Exploration(task.ExplorationID), "worker") + tools, _, cleanup := agent.AugmentTools(ctx, "worker", ts.WorkerTools()) + defer cleanup() + recordFact := names(tools)["record_fact"] + if recordFact == nil { + t.Fatal("worker has no record_fact") + } + if !acceptsRawInput(recordFact) { + t.Errorf("record_fact lost the harness opt-out behind %T", recordFact) + } + + // The first call is malformed (facts is not an array): the harness + // schema check answers it with its generic error unless the tool opted + // out. The second is the legacy single fact, which the published + // schema does not describe and the handler still accepts. + p := &toolScript{results: make(chan string, 4), calls: []scriptedCall{ + {"record_fact", `{"facts":{"summary":"not an array"}}`}, + {"record_fact", `{"summary":"legacy single fact"}`}, + }} + for _, err := range harness.Query(ctx, harness.QueryInput{ + Provider: p, + Tools: actool.NewRegistry(recordFact), + Messages: []llm.Message{llm.UserText("record the facts")}, + }, harness.QueryDeps{}) { + if err != nil { + t.Fatal(err) + } + } + if refusal := p.next(t); !strings.Contains(refusal, "invalid record_fact input: facts must be an array") { + t.Errorf("malformed call: want the handler's own refusal, not the SDK schema error, got: %s", refusal) + } + if legacy := p.next(t); strings.HasPrefix(legacy, "[error]") || ts.Writes().Facts != 1 { + t.Errorf("legacy single fact not written (facts=%d): %s", ts.Writes().Facts, legacy) + } + }) +} diff --git a/server/tool_usage.go b/server/tool_usage.go index 5f9e7c884..5e69cfb45 100644 --- a/server/tool_usage.go +++ b/server/tool_usage.go @@ -33,6 +33,14 @@ func meterTool(t actool.CoreTool, recorder toolUsageRecorder, toolKey, agentKey return &meteredTool{CoreTool: t, recorder: recorder, toolKey: toolKey, agentKey: agentKey, ri: ri} } +// AcceptsRawInput forwards the resolved tool's opt-out of the harness schema +// check; the embedded interface alone would hide it from the harness's type +// assertion. +func (m *meteredTool) AcceptsRawInput() bool { + r, ok := m.CoreTool.(interface{ AcceptsRawInput() bool }) + return ok && r.AcceptsRawInput() +} + func (m *meteredTool) Call(ctx context.Context, input json.RawMessage, tc *actool.ToolContext) (actool.Result, error) { if err := m.recorder.InsertToolUsage(&db.ToolUsage{ ToolKey: m.toolKey, From f758b4d964d150ec8723f4be1adc800a5d0f29c7 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Fri, 20 Nov 2026 18:11:42 +0100 Subject: [PATCH 3/6] fix(server): keep the record_fact refresh flag unset when the refresh fails refreshRecordFactContract wrote its one-time flag in a defer, so a start on which GetTool, the schema decode or the UPDATE failed still spent the flag. No later start retried and the database kept the earlier description. The flag is now written once the outcome is known: after the UPDATE succeeded (also when it changed no row), or when the row is not the built-in one. A failed read, decode or UPDATE leaves it unset and the next start retries. Test: TestFix410RefreshFlagNotSetOnFailure (an UPDATE refused by a trigger, and a stored schema that does not decode; the next start refreshes the text and sets the flag). --- server/fact_input_schema.go | 17 ++++++- server/fact_input_schema_test.go | 83 ++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/server/fact_input_schema.go b/server/fact_input_schema.go index 7ca01c1be..0b194e61a 100644 --- a/server/fact_input_schema.go +++ b/server/fact_input_schema.go @@ -20,12 +20,16 @@ import ( // // Operator edits stay and are named in the log. The schema structure itself is // repaired by seedFactInputSchema on every start (RepairRecordFactSchema). +// +// The flag is written only once the outcome is known: the UPDATE succeeded +// (also when it changed no row) or the row is not the built-in one. A failed +// read, decode or UPDATE leaves it unset, so the next start retries. func (s *Server) refreshRecordFactContract() { const flag = "tool_schema_refresh_record_fact_facts_only" if v, _, _ := s.m.pg.GetSetting(flag); v == "true" { return } - defer func() { _ = s.m.pg.SetSetting(flag, "true") }() + done := func() { _ = s.m.pg.SetSetting(flag, "true") } var seed *agent.ToolSeed for _, sd := range agent.BuiltinToolSeeds() { if sd.Key == "record_fact" { @@ -34,7 +38,15 @@ func (s *Server) refreshRecordFactContract() { } } row, err := s.m.pg.GetTool("record_fact") - if err != nil || row == nil || seed == nil || !row.System || row.Kind != "builtin" { + if err != nil { + log.Printf("[facts] read record_fact for the contract refresh: %v", err) + return + } + if row == nil || seed == nil { + return + } + if !row.System || row.Kind != "builtin" { + done() return } legacyTool, legacyFacts, legacyItems := agent.RecordFactLegacyDescriptions() @@ -76,6 +88,7 @@ WHERE key='record_fact' AND system AND kind='builtin' log.Printf("[facts] refresh record_fact contract: %v", err) return } + done() log.Printf("[facts] record_fact now publishes the facts-only contract (one-time)") } diff --git a/server/fact_input_schema_test.go b/server/fact_input_schema_test.go index 9cf9df34a..0425e7d87 100644 --- a/server/fact_input_schema_test.go +++ b/server/fact_input_schema_test.go @@ -241,3 +241,86 @@ func TestRefreshRecordFactContract(t *testing.T) { t.Fatalf("schema not repaired beside an operator description: %s", row.Schema) } } + +// A start on which the refresh fails must not spend the one-time flag: the +// next start retries and the stored default text is refreshed then. +func TestFix410RefreshFlagNotSetOnFailure(t *testing.T) { + skipWithoutDSN(t) + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Skipf("postgres unavailable (%v)", err) + } + defer m.Close() + s := &Server{m: m} + const flag = "tool_schema_refresh_record_fact_facts_only" + seed := recordFactSeed(t) + seedSchema, _ := json.Marshal(seed.Schema) + seedAgents, _ := json.Marshal(seed.Agents) + legacyTool, _, _ := agent.RecordFactLegacyDescriptions() + legacySchema, _ := recordFactLegacySchema(t) + exec := func(query string, args ...any) { + t.Helper() + if _, err := m.pg.Exec(query, args...); err != nil { + t.Fatal(err) + } + } + const dropTrigger = `DROP TRIGGER IF EXISTS fix410_fail_update ON tools` + defer func() { + _, _ = m.pg.Exec(dropTrigger) + _, _ = m.pg.Exec(`DROP FUNCTION IF EXISTS fix410_fail_update()`) + _ = m.pg.UpsertToolForce("record_fact", seed.Desc, seedSchema, seedAgents) + _, _ = m.pg.Exec(`DELETE FROM settings WHERE key=$1`, flag) + }() + exec(`CREATE OR REPLACE FUNCTION fix410_fail_update() RETURNS trigger LANGUAGE plpgsql AS +'BEGIN RAISE EXCEPTION ''fix410: simulated transient failure''; END'`) + + start := func() { + s.seedFactInputSchema() + s.seedFactSourceSchema() + s.refreshRecordFactContract() + } + for _, tc := range []struct { + name string + fail, clear func() + }{ + {"update fails", func() { + exec(`CREATE TRIGGER fix410_fail_update BEFORE UPDATE ON tools +FOR EACH ROW WHEN (OLD.key = 'record_fact') EXECUTE FUNCTION fix410_fail_update()`) + }, func() { exec(dropTrigger) }}, + {"schema does not decode", func() { + exec(`UPDATE tools SET schema='[]'::jsonb WHERE key='record_fact'`) + }, func() { + exec(`UPDATE tools SET schema=$1::jsonb WHERE key='record_fact'`, legacySchema) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + // A database that stores the earlier default text and has not run the refresh. + if err := m.pg.UpsertToolForce("record_fact", legacyTool[0], json.RawMessage(legacySchema), seedAgents); err != nil { + t.Fatal(err) + } + exec(`DELETE FROM settings WHERE key=$1`, flag) + + tc.fail() + start() + if v, ok, _ := m.pg.GetSetting(flag); ok { + t.Errorf("flag is %q after a failed refresh: later starts skip it", v) + } + if row, _ := m.pg.GetTool("record_fact"); row == nil || row.Description != legacyTool[0] { + t.Fatal("the simulated failure did not leave the earlier description in place") + } + + tc.clear() + start() + row, err := m.pg.GetTool("record_fact") + if err != nil || row == nil { + t.Fatalf("row: %v", err) + } + if row.Description != seed.Desc { + t.Errorf("the next start did not refresh the description: %.60q", row.Description) + } + if v, _, _ := m.pg.GetSetting(flag); v != "true" { + t.Errorf("flag = %q after a successful refresh, want true", v) + } + }) + } +} From 5e1c55db4d67c0fe45c49379b6c319faf061c3cb Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Fri, 20 Nov 2026 18:48:06 +0100 Subject: [PATCH 4/6] fix(server): recognise the released record_fact descriptions as known defaults The releases v0.4.0 to v0.6.0 seed a record_fact description without the source bullet and their own facts property description. Neither was in the list of known defaults, so the one-time refresh kept both texts on a database seeded by a release and logged the description as an operator edit. Both released texts are now known defaults, taken byte for byte from agent/tools.go at those tags, and the log line says the description does not match a known default. --- agent/fact_input_legacy.go | 26 ++++++- server/fact_input_released_test.go | 117 +++++++++++++++++++++++++++++ server/fact_input_schema.go | 4 +- server/fact_input_schema_test.go | 2 +- 4 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 server/fact_input_released_test.go diff --git a/agent/fact_input_legacy.go b/agent/fact_input_legacy.go index 769c53977..e43e24fe8 100644 --- a/agent/fact_input_legacy.go +++ b/agent/fact_input_legacy.go @@ -16,6 +16,17 @@ const recordFactLegacyDescription = "Write an exploration [fact/conclusion] into " · source = where the content came from: target_response (content the target itself returned: page bodies, headers, error messages, banners, even when a tool fetched it) | tool_output (a tool's own result about the target: port states, scanner verdicts, status codes, timings) | third_party_intel (OSINT, CVE databases, public reports) | operator_input (given by the human operator) | inference (your own reasoning; the default when omitted). Any other value is refused.\n" + " · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen." +// recordFactReleasedDescription is the record_fact description seeded by the +// releases v0.4.0 to v0.6.0 (agent/tools.go at those tags): the text above +// without the source bullet. +const recordFactReleasedDescription = "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n" + + "⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n" + + "★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n" + + "⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n" + + " · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n" + + " · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n" + + " · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen." + // recordFactLegacyItemDescriptions are the facts[] item descriptions of the // earlier contract whose wording referred to the top level; the refresh // replaces exactly these texts with the current ones. @@ -28,9 +39,16 @@ var recordFactLegacyItemDescriptions = map[string]string{ // earlier contract. const recordFactLegacyFactsDescription = "A non-empty batch of complete facts. Every item needs its own summary. Only intent_id inherits from the top level. Returned ids match item order; retry only failed indices." +// recordFactReleasedFactsDescription is the facts property description seeded +// by the releases v0.4.0 to v0.6.0. RepairRecordFactSchema keeps a stored facts +// description, so the refresh has to know this text to replace it. +const recordFactReleasedFactsDescription = "[use when there are several distinct conclusions] array of facts; element fields same as the top-level fields below (summary/detail/evidence/confidence/intent_id/asset_ids); omitting intent_id falls back to the top-level intent_id. The returned ids match this array in length and order." + // RecordFactLegacyDescriptions returns the description texts of the earlier -// contract that the refresh may replace: the tool description, the facts -// property description and the item descriptions. -func RecordFactLegacyDescriptions() (tool []string, facts string, items map[string]string) { - return []string{recordFactLegacyDescription}, recordFactLegacyFactsDescription, recordFactLegacyItemDescriptions +// contract that the refresh may replace: the tool descriptions, the facts +// property descriptions and the item descriptions. +func RecordFactLegacyDescriptions() (tool, facts []string, items map[string]string) { + return []string{recordFactLegacyDescription, recordFactReleasedDescription}, + []string{recordFactLegacyFactsDescription, recordFactReleasedFactsDescription}, + recordFactLegacyItemDescriptions } diff --git a/server/fact_input_released_test.go b/server/fact_input_released_test.go new file mode 100644 index 000000000..4d2116e69 --- /dev/null +++ b/server/fact_input_released_test.go @@ -0,0 +1,117 @@ +package server + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "testing" +) + +// The record_fact texts seeded by the releases v0.4.0 to v0.6.0, copied from +// agent/tools.go at those tags. They are kept here, apart from the agent +// package's list of known defaults, so the test fails when that list drifts +// from the released bytes. +const fix410ReleasedDescription = "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n" + + "⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n" + + "★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n" + + "⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n" + + " · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n" + + " · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n" + + " · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen." + +const fix410ReleasedFactsDescription = "[use when there are several distinct conclusions] array of facts; element fields same as the top-level fields below (summary/detail/evidence/confidence/intent_id/asset_ids); omitting intent_id falls back to the top-level intent_id. The returned ids match this array in length and order." + +// A database seeded by a release stores the released record_fact description, +// the released facts property description and facts items without properties. +// The one-time refresh replaces both texts with the current ones and sets its +// flag; texts an operator edited match no known default and stay as stored. +func TestFix410ReleasedDefaultsRefreshed(t *testing.T) { + for _, c := range []struct { + name, text, sum string + size int + }{ + {"description", fix410ReleasedDescription, "0da5cec5cde7f5e6f41343418fcc2a7d132d6520f336efb39fcbf1ab6ac51957", 2038}, + {"facts description", fix410ReleasedFactsDescription, "eb054843a6b33dd39b94ba0648f292babb9cf9bdd106ee4eff6bf593d3f3edf4", 286}, + } { + sum := sha256.Sum256([]byte(c.text)) + if len(c.text) != c.size || hex.EncodeToString(sum[:]) != c.sum { + t.Fatalf("the released %s in this test is not the released text: %d bytes, sha256 %x", c.name, len(c.text), sum) + } + } + skipWithoutDSN(t) + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + s := &Server{m: m} + const flag = "tool_schema_refresh_record_fact_facts_only" + seed := recordFactSeed(t) + seedSchema, _ := json.Marshal(seed.Schema) + seedAgents, _ := json.Marshal(seed.Agents) + defer func() { + _ = m.pg.UpsertToolForce("record_fact", seed.Desc, seedSchema, seedAgents) + _, _ = m.pg.Exec(`DELETE FROM settings WHERE key=$1`, flag) + }() + currentFacts := seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)["description"] + + // start stores the row as a release seeded it, with the given two texts, + // runs the startup steps and returns the two texts as stored afterwards. + start := func(desc, factsDesc string) (string, any) { + t.Helper() + str := func(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + schema, _ := json.Marshal(map[string]any{"type": "object", "properties": map[string]any{ + "facts": map[string]any{"type": "array", "description": factsDesc, "items": map[string]any{"type": "object"}}, + "summary": str("a [one-sentence summary] of this exploration's conclusion (a condensation of detail)"), + "intent_id": map[string]any{"type": "integer", "description": "the id of the intent that produced this fact (the intent you claimed; the default for each item in batch mode)"}, + "detail": str("details related to this fact: write all of this exploration's observations here"), + "evidence": str("[one line] of key evidence: the command + the one or two most proving output lines. Keep it brief; don't paste large output blocks (details go in detail)."), + "confidence": str("observed (seen directly in output) | inferred (inferred from phenomena). Always label negative conclusions honestly."), + "asset_ids": map[string]any{"type": "array", "items": map[string]any{"type": "integer"}, "description": "related asset ids (optional, 0/1/many): which assets this fact involves"}, + }}) + if err := m.pg.UpsertToolForce("record_fact", desc, schema, seedAgents); err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec(`DELETE FROM settings WHERE key=$1`, flag); err != nil { + t.Fatal(err) + } + s.seedFactInputSchema() + s.seedFactSourceSchema() + s.refreshRecordFactContract() + if v, _, _ := m.pg.GetSetting(flag); v != "true" { + t.Errorf("flag = %q after the refresh, want true", v) + } + row, err := m.pg.GetTool("record_fact") + if err != nil || row == nil { + t.Fatalf("row: %v", err) + } + var stored struct { + Properties map[string]map[string]any `json:"properties"` + } + if err := json.Unmarshal(row.Schema, &stored); err != nil { + t.Fatal(err) + } + if len(stored.Properties) != 1 || stored.Properties["facts"] == nil { + t.Fatalf("top level still holds fact fields: %s", row.Schema) + } + return row.Description, stored.Properties["facts"]["description"] + } + + desc, factsDesc := start(fix410ReleasedDescription, fix410ReleasedFactsDescription) + if desc != seed.Desc { + t.Errorf("released description not replaced by the current one: %.60q", desc) + } + if factsDesc != currentFacts { + t.Errorf("released facts description not replaced by the current one: %v", factsDesc) + } + + editedDesc := fix410ReleasedDescription + "\nOperator note: record every negative result." + editedFacts := fix410ReleasedFactsDescription + " Operator note: one fact per host." + desc, factsDesc = start(editedDesc, editedFacts) + if desc != editedDesc { + t.Errorf("operator description changed: %.60q", desc) + } + if factsDesc != editedFacts { + t.Errorf("operator facts description changed: %v", factsDesc) + } +} diff --git a/server/fact_input_schema.go b/server/fact_input_schema.go index 0b194e61a..d6c3a1005 100644 --- a/server/fact_input_schema.go +++ b/server/fact_input_schema.go @@ -56,7 +56,7 @@ func (s *Server) refreshRecordFactContract() { case sliceHas(legacyTool, desc): desc = seed.Desc default: - log.Printf("[facts] record_fact description was edited by an operator and still describes the earlier contract; the facts-only schema is published anyway, review the text in the tool catalog") + log.Printf("[facts] record_fact description does not match a known default and still describes the earlier contract; the facts-only schema is published anyway, review the text in the tool catalog") } var schema map[string]any if err := json.Unmarshal(row.Schema, &schema); err != nil { @@ -66,7 +66,7 @@ func (s *Server) refreshRecordFactContract() { repaired := agent.RepairRecordFactSchema(schema) facts := repaired["properties"].(map[string]any)["facts"].(map[string]any) canonFacts := seed.Schema["properties"].(map[string]any)["facts"].(map[string]any) - if facts["description"] == legacyFacts { + if d, ok := facts["description"].(string); ok && sliceHas(legacyFacts, d) { facts["description"] = canonFacts["description"] } items := facts["items"].(map[string]any)["properties"].(map[string]any) diff --git a/server/fact_input_schema_test.go b/server/fact_input_schema_test.go index 0425e7d87..959697147 100644 --- a/server/fact_input_schema_test.go +++ b/server/fact_input_schema_test.go @@ -129,7 +129,7 @@ func recordFactLegacySchema(t *testing.T) (string, map[string]string) { "intent_id": map[string]any{"type": "integer", "description": legacyItems["intent_id"]}, "detail": map[string]any{"type": "string", "description": "Details of this fact, including all related observations."}, "source": map[string]any{"type": "string", "enum": []string{"inference"}}, - "facts": map[string]any{"type": "array", "minItems": 1, "description": legacyFacts, "items": item}, + "facts": map[string]any{"type": "array", "minItems": 1, "description": legacyFacts[0], "items": item}, }} raw, _ := json.Marshal(schema) return string(raw), legacyItems From aa8ba54efb3f7edbfad50b57a91767b14f5bfae9 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Fri, 20 Nov 2026 19:13:38 +0100 Subject: [PATCH 5/6] fix(server): reword the record_fact default-mismatch log line --- server/fact_input_schema.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/fact_input_schema.go b/server/fact_input_schema.go index d6c3a1005..173fb0b8f 100644 --- a/server/fact_input_schema.go +++ b/server/fact_input_schema.go @@ -56,7 +56,7 @@ func (s *Server) refreshRecordFactContract() { case sliceHas(legacyTool, desc): desc = seed.Desc default: - log.Printf("[facts] record_fact description does not match a known default and still describes the earlier contract; the facts-only schema is published anyway, review the text in the tool catalog") + log.Printf("[facts] record_fact description does not match a known default (operator edit or an older release); the facts-only schema is published anyway, review the text in the tool catalog") } var schema map[string]any if err := json.Unmarshal(row.Schema, &schema); err != nil { From 5def7bf57ebc7b5f3b1939f9db9d097c14b3ef35 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Fri, 20 Nov 2026 20:07:50 +0100 Subject: [PATCH 6/6] docs(server): correct the record_fact refresh comment --- server/fact_input_schema.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/fact_input_schema.go b/server/fact_input_schema.go index 173fb0b8f..31ad152e0 100644 --- a/server/fact_input_schema.go +++ b/server/fact_input_schema.go @@ -18,8 +18,9 @@ import ( // - the tool description; // - the facts property and item descriptions that referred to the top level. // -// Operator edits stay and are named in the log. The schema structure itself is -// repaired by seedFactInputSchema on every start (RepairRecordFactSchema). +// Operator edits stay; a description that matches no known default is logged +// as such and left in place. The schema structure itself is repaired by +// seedFactInputSchema on every start (RepairRecordFactSchema). // // The flag is written only once the outcome is known: the UPDATE succeeded // (also when it changed no row) or the row is not the built-in one. A failed