diff --git a/agent/assembly.go b/agent/assembly.go index c0885d94e..4a075e79d 100644 --- a/agent/assembly.go +++ b/agent/assembly.go @@ -101,6 +101,14 @@ func guardPanic(t actool.CoreTool) actool.CoreTool { return &guardedTool{CoreToo type guardedTool struct{ actool.CoreTool } +// AcceptsRawInput forwards the wrapped tool's opt-out of the harness schema +// check, as overriddenTool does: this wrapper is the one the harness sees, and +// the embedded interface alone would hide the opt-out from its type assertion. +func (g *guardedTool) AcceptsRawInput() bool { + r, ok := g.CoreTool.(interface{ AcceptsRawInput() bool }) + return ok && r.AcceptsRawInput() +} + func (g *guardedTool) Call(ctx context.Context, in json.RawMessage, tc *actool.ToolContext) (res actool.Result, err error) { defer func() { if r := recover(); r != nil { diff --git a/agent/fact_assets_test.go b/agent/fact_assets_test.go index d2b1a7838..ec67e0271 100644 --- a/agent/fact_assets_test.go +++ b/agent/fact_assets_test.go @@ -115,10 +115,9 @@ func TestRecordFactAssetAliases(t *testing.T) { func TestRecordFactAssetAliasSchema(t *testing.T) { schema := NewToolSet(nil, "").recordFact().InputSchema() props := schema["properties"].(map[string]any) - for _, name := range []string{"asset_ids", "tested_asset_ids"} { - if props[name] == nil { - t.Fatalf("single schema missing %s", name) - } + // Facts-only contract: the alias lives on the items. + if len(props) != 1 || props["facts"] == nil { + t.Fatalf("top level must hold facts only: %v", props) } item := props["facts"].(map[string]any)["items"].(map[string]any) itemProps, _ := item["properties"].(map[string]any) diff --git a/agent/fact_input.go b/agent/fact_input.go index 60c08902d..3b6581b8d 100644 --- a/agent/fact_input.go +++ b/agent/fact_input.go @@ -4,10 +4,12 @@ import ( "bytes" "encoding/json" "fmt" + "sort" + "strings" ) // Match strings.TrimSpace, including Unicode whitespace, in the published schema. -const factWhitespace = "\t\n\v\f\r \u0085\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000" +const factWhitespace = "\t\n\v\f\r \u0085   - 

   " func factAssetIDSchema() map[string]any { return map[string]any{ @@ -16,10 +18,13 @@ func factAssetIDSchema() map[string]any { } } +// factItemSchema is one fact: the shape of every facts[] item. It is also the +// shape of the legacy single-fact call (fields at the top level, no facts), +// which the handler still accepts but the published schema no longer shows. func factItemSchema() map[string]any { schema := obj(map[string]any{ - "summary": str("Required on every fact: a non-empty one-sentence summary of this conclusion. Batch items do not inherit summary from the top level."), - "intent_id": idp("The intent that produced this fact; only this field inherits from the top level in a batch."), + "summary": str("Required: a non-empty one-sentence summary of this conclusion."), + "intent_id": idp("The intent that produced this fact. Omit it to link the fact to the intent this worker is running; a batch may name a different intent per item."), "detail": str("Details of this fact, including all related observations."), "evidence": str("One line of key evidence: the command and the most proving output lines. Put longer output in detail."), "confidence": str("observed (seen directly in output) | inferred (inferred from phenomena). Label negative conclusions honestly."), @@ -34,66 +39,65 @@ func factItemSchema() map[string]any { return schema } +// recordFactFactsDescription documents the only top-level property of +// record_fact. The wording tells a model that tends to fill every declared +// property that there is nothing else to fill. +const recordFactFactsDescription = "The facts to record, usually exactly one. Every item is a complete fact with its own summary. This is the only top-level property: never send summary, detail, evidence, confidence, source, intent_id or asset ids beside facts. Returned ids match item order; retry only failed indices." + +// recordFactContractNote is appended to the record_fact description: the one +// call shape the model is given. +const recordFactContractNote = "\n★Call shape: exactly {\"facts\":[{...}]} with one item per distinct conclusion (usually one item). Each item carries its own summary, detail, evidence, confidence, source, asset_ids and, when it differs from the intent you are running, intent_id. Nothing else at the top level; a top-level fact field beside facts is refused and nothing is written." + +// RecordFactContractNote exposes the text to the server's one-time refresh. +func RecordFactContractNote() string { return recordFactContractNote } + +// recordFactSchema is the published input contract: facts[] and nothing else. +// Providers reject root-level anyOf, and the earlier contract (fact fields at +// the top level beside facts[]) made models fill both; the handler alone keeps +// the legacy shapes alive. func recordFactSchema() map[string]any { - props := factItemSchema()["properties"].(map[string]any) - props["facts"] = map[string]any{ - "type": "array", "minItems": 1, "items": factItemSchema(), - "description": "A non-empty batch of complete facts. Every item needs its own summary. Only intent_id inherits from the top level. Returned ids match item order; retry only failed indices.", - } - schema := obj(props) - // Providers reject root-level anyOf. The handler enforces single/batch mode. + schema := obj(map[string]any{ + "facts": map[string]any{ + "type": "array", "minItems": 1, "items": factItemSchema(), + "description": recordFactFactsDescription, + }, + }, "facts") schema["additionalProperties"] = false return schema } -// RepairRecordFactSchema restores the single/batch input contract in stored or -// cached schemas, preserving annotations only on fields the handler accepts. +// RepairRecordFactSchema brings a stored or cached record_fact schema to the +// facts-only contract. The structure always comes from the code; of the stored +// schema only the annotations (description, default) of facts and of its known +// item fields survive. Top-level fact fields of the earlier contract are +// dropped: the handler still accepts them, but the model must not see them. func RepairRecordFactSchema(stored map[string]any) map[string]any { - raw, _ := json.Marshal(stored) - var out map[string]any - _ = json.Unmarshal(raw, &out) - if out == nil { - out = map[string]any{} + out := recordFactSchema() + facts := out["properties"].(map[string]any)["facts"].(map[string]any) + storedProps, _ := stored["properties"].(map[string]any) + storedFacts, _ := storedProps["facts"].(map[string]any) + if storedFacts == nil { + return out } - mergeFactContract(out, recordFactSchema()) - // A batch requires summaries on its items, never a redundant top-level one. - delete(out, "required") - delete(out, "anyOf") - canonical := recordFactSchema()["properties"].(map[string]any) - props := out["properties"].(map[string]any) - for name := range props { - if _, known := canonical[name]; !known { - delete(props, name) - } + if d, ok := storedFacts["description"].(string); ok && strings.TrimSpace(d) != "" { + facts["description"] = d } - items := props["facts"].(map[string]any)["items"].(map[string]any) - itemProps := items["properties"].(map[string]any) - for name := range itemProps { - if _, known := canonical[name]; !known || name == "facts" { - delete(itemProps, name) - } - } - return out -} - -func mergeFactContract(dst, src map[string]any) { - for key, value := range src { - if key == "description" || key == "default" { - if _, ok := dst[key]; ok { - continue - } + storedItems, _ := storedFacts["items"].(map[string]any) + storedItemProps, _ := storedItems["properties"].(map[string]any) + itemProps := facts["items"].(map[string]any)["properties"].(map[string]any) + for name, p := range itemProps { + sp, _ := storedItemProps[name].(map[string]any) + if sp == nil { + continue } - if child, ok := value.(map[string]any); ok { - target, _ := dst[key].(map[string]any) - if target == nil { - target = map[string]any{} - dst[key] = target + prop := p.(map[string]any) + for _, key := range []string{"description", "default"} { + if v, ok := sp[key]; ok && v != nil { + prop[key] = v } - mergeFactContract(target, child) - } else { - dst[key] = value } } + return out } type recordFactInput struct { @@ -103,6 +107,9 @@ type recordFactInput struct { // Validate exact field names before decoding: encoding/json otherwise treats // Summary, SUMMARY, etc. as aliases even with DisallowUnknownFields enabled. +// Three input shapes reach the handler: the published one ({facts:[...]}), the +// legacy single fact (fields at the top level, no facts) and the legacy batch +// with a top-level intent_id as the default of its items. func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) { var out recordFactInput fields, err := exactFactFields(in, true) @@ -111,11 +118,6 @@ func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) { } _, batch := fields["facts"] if batch { - for name := range fields { - if name != "intent_id" && name != "facts" { - return out, true, fmt.Errorf("field %q is not allowed at the top level in batch mode; only intent_id and facts are accepted", name) - } - } var items []json.RawMessage if err := json.Unmarshal(fields["facts"], &items); err != nil { return out, true, fmt.Errorf("facts must be an array: %w", err) @@ -123,10 +125,16 @@ func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) { if len(items) == 0 { return out, true, fmt.Errorf("facts must be a non-empty array of complete facts, each with its own summary") } + itemFields := make([]map[string]json.RawMessage, len(items)) for i, item := range items { - if _, err := exactFactFields(item, false); err != nil { + f, err := exactFactFields(item, false) + if err != nil { return out, true, fmt.Errorf("facts[%d]: %w", i, err) } + itemFields[i] = f + } + if err := checkBatchTopLevel(fields, itemFields); err != nil { + return out, true, err } } decoder := json.NewDecoder(bytes.NewReader(in)) @@ -137,6 +145,28 @@ func decodeRecordFactInput(in json.RawMessage) (recordFactInput, bool, error) { return out, batch, nil } +// checkBatchTopLevel enforces the contract at the top level of a facts[] call, +// where only facts and (legacy) intent_id carry meaning. Any other top-level +// field is refused by name before anything is written, including empty +// placeholders and copies of a fact's fields: a value there is at best +// redundant and at worst a second, conflicting claim. The message names the +// one accepted shape, so a model that filled every declared property of an +// older schema learns where the fields belong. +func checkBatchTopLevel(fields map[string]json.RawMessage, items []map[string]json.RawMessage) error { + var extra []string + for name := range fields { + if name == "intent_id" || name == "facts" { + continue + } + extra = append(extra, fmt.Sprintf("%q", name)) + } + if len(extra) == 0 { + return nil + } + sort.Strings(extra) + return fmt.Errorf("top-level field(s) %s are not allowed beside facts (%d item(s)): record_fact takes only {\"facts\":[...]}; put every field of a fact inside its facts item and send nothing else at the top level", strings.Join(extra, ", "), len(items)) +} + func exactFactFields(in json.RawMessage, topLevel bool) (map[string]json.RawMessage, error) { var fields map[string]json.RawMessage if err := json.Unmarshal(in, &fields); err != nil { diff --git a/agent/fact_input_legacy.go b/agent/fact_input_legacy.go new file mode 100644 index 000000000..e43e24fe8 --- /dev/null +++ b/agent/fact_input_legacy.go @@ -0,0 +1,54 @@ +package agent + +// Texts of the earlier record_fact contract (fact fields at the top level +// beside facts[]), kept verbatim so the server's one-time refresh +// (server/fact_input_schema.go) can tell a stored default from an operator +// edit. Nothing here is published to a model. + +// recordFactLegacyDescription is the record_fact description seeded up to the +// facts-only contract. +const recordFactLegacyDescription = "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n" + + "⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n" + + "★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n" + + "⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n" + + " · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n" + + " · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n" + + " · source = where the content came from: target_response (content the target itself returned: page bodies, headers, error messages, banners, even when a tool fetched it) | tool_output (a tool's own result about the target: port states, scanner verdicts, status codes, timings) | third_party_intel (OSINT, CVE databases, public reports) | operator_input (given by the human operator) | inference (your own reasoning; the default when omitted). Any other value is refused.\n" + + " · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen." + +// recordFactReleasedDescription is the record_fact description seeded by the +// releases v0.4.0 to v0.6.0 (agent/tools.go at those tags): the text above +// without the source bullet. +const recordFactReleasedDescription = "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n" + + "⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n" + + "★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n" + + "⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n" + + " · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n" + + " · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n" + + " · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen." + +// recordFactLegacyItemDescriptions are the facts[] item descriptions of the +// earlier contract whose wording referred to the top level; the refresh +// replaces exactly these texts with the current ones. +var recordFactLegacyItemDescriptions = map[string]string{ + "summary": "Required on every fact: a non-empty one-sentence summary of this conclusion. Batch items do not inherit summary from the top level.", + "intent_id": "The intent that produced this fact; only this field inherits from the top level in a batch.", +} + +// recordFactLegacyFactsDescription is the facts property description of the +// earlier contract. +const recordFactLegacyFactsDescription = "A non-empty batch of complete facts. Every item needs its own summary. Only intent_id inherits from the top level. Returned ids match item order; retry only failed indices." + +// recordFactReleasedFactsDescription is the facts property description seeded +// by the releases v0.4.0 to v0.6.0. RepairRecordFactSchema keeps a stored facts +// description, so the refresh has to know this text to replace it. +const recordFactReleasedFactsDescription = "[use when there are several distinct conclusions] array of facts; element fields same as the top-level fields below (summary/detail/evidence/confidence/intent_id/asset_ids); omitting intent_id falls back to the top-level intent_id. The returned ids match this array in length and order." + +// RecordFactLegacyDescriptions returns the description texts of the earlier +// contract that the refresh may replace: the tool descriptions, the facts +// property descriptions and the item descriptions. +func RecordFactLegacyDescriptions() (tool, facts []string, items map[string]string) { + return []string{recordFactLegacyDescription, recordFactReleasedDescription}, + []string{recordFactLegacyFactsDescription, recordFactReleasedFactsDescription}, + recordFactLegacyItemDescriptions +} diff --git a/agent/fact_input_test.go b/agent/fact_input_test.go index 6c02c128b..7b48a1356 100644 --- a/agent/fact_input_test.go +++ b/agent/fact_input_test.go @@ -3,9 +3,11 @@ package agent import ( "encoding/json" "regexp" + "strconv" "strings" "testing" + "github.com/Autumn-27/artex/db" actool "github.com/Autumn-27/norma/tool" ) @@ -47,6 +49,10 @@ func TestRecordFactSummaryContract(t *testing.T) { } } +// A stored schema of the earlier contract (fact fields at the top level beside +// facts[], operator annotations, a stray property) is republished as the +// facts-only contract; annotations of facts and of its items survive, the top +// level loses every fact field, and the result is idempotent. func TestRecordFactStoredContractRepair(t *testing.T) { old := map[string]any{ "type": "object", "required": []any{"summary"}, @@ -55,32 +61,40 @@ func TestRecordFactStoredContractRepair(t *testing.T) { "summary": map[string]any{"type": "string", "description": "operator summary"}, "confidence": map[string]any{"type": "string", "default": "inferred"}, "custom_field": map[string]any{"type": "string"}, - "facts": map[string]any{"type": "array", "items": map[string]any{"type": "object"}}, + "facts": map[string]any{"type": "array", "description": "operator facts", "items": map[string]any{"type": "object", "properties": map[string]any{ + "summary": map[string]any{"type": "string", "description": "operator item summary"}, + "confidence": map[string]any{"type": "string", "default": "observed"}, + "custom": map[string]any{"type": "string"}, + }}}, }, } base := NewToolSet(nil, "").recordFact() repaired := DecorateTool(base, "operator tool", old) schema := repaired.InputSchema() - if _, required := schema["required"]; required { - t.Fatal("batch still requires a top-level summary") + if req, _ := schema["required"].([]string); len(req) != 1 || req[0] != "facts" { + t.Fatalf("facts must be the only required property: %v", schema["required"]) } if _, exists := schema["anyOf"]; exists { t.Fatal("root anyOf reaches the provider") } props := schema["properties"].(map[string]any) - item := props["facts"].(map[string]any)["items"].(map[string]any) + if len(props) != 1 || props["facts"] == nil || schema["additionalProperties"] != false { + t.Fatalf("top level must hold facts only: %v", props) + } + facts := props["facts"].(map[string]any) + item := facts["items"].(map[string]any) required := item["required"].([]string) - if len(required) != 1 || required[0] != "summary" || props["facts"].(map[string]any)["minItems"] != 1 { + if len(required) != 1 || required[0] != "summary" || facts["minItems"] != 1 { t.Fatal("batch must require a nonempty list with a summary per item") } - if props["summary"].(map[string]any)["minLength"] != 1 || - item["properties"].(map[string]any)["summary"].(map[string]any)["minLength"] != 1 { + itemProps := item["properties"].(map[string]any) + if itemProps["summary"].(map[string]any)["minLength"] != 1 { t.Fatal("empty summaries permitted by schema") } - if repaired.Description() != "operator tool" || - props["summary"].(map[string]any)["description"] != "operator summary" || - props["confidence"].(map[string]any)["default"] != "inferred" || props["custom_field"] != nil { - t.Fatal("operator annotations lost") + if repaired.Description() != "operator tool" || facts["description"] != "operator facts" || + itemProps["summary"].(map[string]any)["description"] != "operator item summary" || + itemProps["confidence"].(map[string]any)["default"] != "observed" || itemProps["custom"] != nil { + t.Fatalf("item annotations lost or stray property kept: %v", itemProps) } if _, ok := old["properties"].(map[string]any)["tested_asset_ids"]; ok { t.Fatal("shared old schema mutated") @@ -94,9 +108,18 @@ func TestRecordFactStoredContractRepair(t *testing.T) { if err := json.Unmarshal(first, &roundTrip); err != nil { t.Fatal(err) } - for _, input := range []string{`{"summary":"single"}`, `{"facts":[{"summary":"batch"}]}`} { - if err := actool.ValidateInput(roundTrip, json.RawMessage(input)); err != nil { - t.Fatalf("SDK rejects valid %s after catalog round trip: %v", input, err) + if err := actool.ValidateInput(roundTrip, json.RawMessage(`{"facts":[{"summary":"batch"}]}`)); err != nil { + t.Fatalf("SDK rejects the published shape after catalog round trip: %v", err) + } + // The legacy single shape no longer passes the published schema; the + // handler validates its own input instead, also behind the catalog wrapper. + if err := actool.ValidateInput(roundTrip, json.RawMessage(`{"summary":"single"}`)); err == nil { + t.Fatal("published schema must require facts") + } + for _, tool := range []actool.CoreTool{base, repaired} { + r, ok := tool.(interface{ AcceptsRawInput() bool }) + if !ok || !r.AcceptsRawInput() { + t.Fatalf("%T must opt out of the harness schema check", tool) } } } @@ -140,7 +163,7 @@ func TestRecordFactStrictFields(t *testing.T) { } func TestRecordFactSchemaMatchesStrings(t *testing.T) { - props := recordFactSchema()["properties"].(map[string]any) + props := factItemSchema()["properties"].(map[string]any) summary := regexp.MustCompile(props["summary"].(map[string]any)["pattern"].(string)) for _, value := range []string{"", " ", "\t\r\n", "\u00a0\u0085", "\u2003\u3000", "conclusion", " observed "} { if got, want := summary.MatchString(value), strings.TrimSpace(value) != ""; got != want { @@ -155,7 +178,7 @@ func TestRecordFactSchemaMatchesStrings(t *testing.T) { } } for _, input := range []string{ - `{"summary":"single","asset_ids":["1",2]}`, + `{"facts":[{"summary":"batch","asset_ids":["1",2]}]}`, `{"facts":[{"summary":"batch","tested_asset_ids":["1",2]}]}`, } { if err := actool.ValidateInput(recordFactSchema(), json.RawMessage(input)); err != nil { @@ -163,3 +186,208 @@ func TestRecordFactSchemaMatchesStrings(t *testing.T) { } } } + +// A model that fills every declared property of the earlier schema sends the +// single fact inside facts[] and a copy (or a placeholder) of every fact field +// beside it. Such a call is refused by name before any write, whether the +// copies are identical, empty or dummies, and the message names the one shape +// that is accepted. The published schema no longer declares those fields. +func TestRecordFactRefusesTopLevelCopiesBesideFacts(t *testing.T) { + store := guardTaskStore(t) + intentID := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "fingerprint the service"}, "running") + tools := NewToolSet(store, "worker") + before, err := store.ListByKind(db.KindFact, 100) + if err != nil { + t.Fatal(err) + } + id := strconv.FormatInt(intentID, 10) + fact := `{"summary":"TLS 1.0 is offered on port 8443","detail":"the handshake succeeded with TLSv1.0","evidence":"openssl s_client -tls1 → Protocol: TLSv1","confidence":"observed","source":"tool_output","intent_id":` + id + `,"asset_ids":[]}` + for _, tc := range []struct{ call, want string }{ + // an exact copy of every field beside facts[] + {`{"summary":"TLS 1.0 is offered on port 8443","detail":"the handshake succeeded with TLSv1.0","evidence":"openssl s_client -tls1 → Protocol: TLSv1","confidence":"observed","source":"tool_output","intent_id":` + id + `,"asset_ids":[],"facts":[` + fact + `]}`, `"asset_ids", "confidence", "detail", "evidence", "source", "summary"`}, + // empty placeholders only + {`{"summary":"","detail":"","evidence":"","confidence":"","source":"","asset_ids":[],"tested_asset_ids":[],"intent_id":` + id + `,"facts":[` + fact + `]}`, `"asset_ids", "confidence", "detail", "evidence", "source", "summary", "tested_asset_ids"`}, + // dummy values + {`{"summary":"x","facts":[` + fact + `]}`, `"summary"`}, + {`{"summary":null,"facts":[` + fact + `]}`, `"summary"`}, + } { + w := tools.Writes().Facts + r := callGuardTool(t, tools.recordFact(), tc.call) + if !r.IsError || !strings.Contains(r.Flatten(), tc.want) || !strings.Contains(r.Flatten(), `record_fact takes only {"facts":[...]}`) || tools.Writes().Facts != w { + t.Errorf("%s: want refusal naming %s, got %s (writes %d→%d)", tc.call, tc.want, r.Flatten(), w, tools.Writes().Facts) + } + } + if n, err := store.ListByKind(db.KindFact, 100); err != nil || len(n) != len(before) { + t.Fatalf("refused calls wrote facts: %d → %d %v", len(before), len(n), err) + } + // The same fact in the published shape is written once and linked. + res := callGuardTool(t, tools.recordFact(), `{"facts":[`+fact+`]}`) + if res.IsError || tools.Writes().Facts != 1 { + t.Fatalf("published shape refused: %s", res.Flatten()) + } + var out struct { + IDs []int64 `json:"ids"` + } + if err := json.Unmarshal([]byte(res.Flatten()), &out); err != nil || len(out.IDs) != 1 || out.IDs[0] <= 0 { + t.Fatalf("batch result must carry exactly one id: %s", res.Flatten()) + } + if yielded, err := store.FactsYielded(intentID); err != nil || len(yielded) != 1 || yielded[0] != out.IDs[0] { + t.Fatalf("fact not linked to the intent: %v %v", yielded, err) + } + node, err := store.GetNode(out.IDs[0]) + if err != nil || node == nil { + t.Fatal(err) + } + var payload map[string]any + _ = json.Unmarshal(node.Payload, &payload) + if payload["summary"] != "TLS 1.0 is offered on port 8443" || payload["source"] != "tool_output" || payload["confidence"] != "observed" { + t.Fatalf("fact payload: %s", node.Payload) + } +} + +// Conflicting copies, placeholders, unknown fields, bad asset ids, missing +// summaries and empty batches are refused, and nothing is written by a +// refused call. +func TestRecordFactBatchTopLevelConflicts(t *testing.T) { + store := guardTaskStore(t) + tools := NewToolSet(store, "worker") + before, err := store.ListByKind(db.KindFact, 100) // the task's origin fact + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ call, want string }{ + // dummy summaries a model may send after a refusal + {`{"summary":"x","facts":[{"summary":"real conclusion"}]}`, `"summary"`}, + {`{"summary":"placeholder","detail":"test","evidence":"test","facts":[{"summary":"real","detail":"real detail","evidence":"real evidence"}]}`, `"detail", "evidence", "summary"`}, + {`{"summary":"contradictory","facts":[{"summary":"different"}]}`, `"summary"`}, + {`{"detail":"invented","facts":[{"summary":"legitimate"}]}`, `"detail"`}, + {`{"confidence":"observed","facts":[{"summary":"fact says inferred","confidence":"inferred"}]}`, `"confidence"`}, + {`{"source":"inference","facts":[{"summary":"fact says tool_output","source":"tool_output"}]}`, `"source"`}, + {`{"asset_ids":[1],"facts":[{"summary":"fact has other assets","asset_ids":[2]}]}`, `"asset_ids"`}, + {`{"summary":"first","facts":[{"summary":"first"},{"summary":"second"}]}`, `(2 item(s))`}, + {`{"summary":"x","typo":1,"facts":[{"summary":"unknown field"}]}`, `"typo"`}, + {`{"facts":[{"summary":"unknown nested","typo":1}]}`, `"typo"`}, + {`{"facts":[{"summary":"bad asset id","asset_ids":["abc"]}]}`, `asset`}, + {`{"summary":"must not inherit","facts":[{"detail":"missing summary"}]}`, `"summary"`}, + {`{"summary":"x","facts":[]}`, `non-empty`}, + {`{"facts":[]}`, `non-empty`}, + {`{"summary":"","facts":[]}`, `non-empty`}, + } { + w := tools.Writes().Facts + r := callGuardTool(t, tools.recordFact(), tc.call) + if !r.IsError || !strings.Contains(r.Flatten(), tc.want) || tools.Writes().Facts != w { + t.Errorf("%s: want refusal mentioning %s, got %s (writes %d→%d)", tc.call, tc.want, r.Flatten(), w, tools.Writes().Facts) + } + } + if n, err := store.ListByKind(db.KindFact, 100); err != nil || len(n) != len(before) { + t.Fatalf("refused calls wrote facts: %d → %d %v", len(before), len(n), err) + } +} + +// The published schema holds facts[] and nothing else, and the description +// names that one shape; the earlier "two shapes" wording is gone. +func TestRecordFactSchemaIsFactsOnly(t *testing.T) { + schema := recordFactSchema() + props := schema["properties"].(map[string]any) + if len(props) != 1 || props["facts"] == nil { + t.Fatalf("top-level properties: %v", props) + } + if req, _ := schema["required"].([]string); len(req) != 1 || req[0] != "facts" || schema["additionalProperties"] != false { + t.Fatalf("facts must be required and the only property: %v %v", schema["required"], schema["additionalProperties"]) + } + facts := props["facts"].(map[string]any) + if d, _ := facts["description"].(string); !strings.Contains(d, "never send summary") { + t.Errorf("facts description must forbid top-level copies: %q", d) + } + item := facts["items"].(map[string]any)["properties"].(map[string]any) + for _, name := range []string{"summary", "intent_id", "detail", "evidence", "confidence", "source", "asset_ids", "tested_asset_ids"} { + if item[name] == nil { + t.Errorf("facts[] item lacks %s", name) + } + } + for name, d := range map[string]string{ + "summary": item["summary"].(map[string]any)["description"].(string), + "intent_id": item["intent_id"].(map[string]any)["description"].(string), + } { + if strings.Contains(d, "top level") || strings.Contains(d, "top-level") { + t.Errorf("facts[].%s still refers to a top level: %q", name, d) + } + } + desc := NewToolSet(nil, "").recordFact().Description() + if !strings.HasSuffix(desc, recordFactContractNote) || strings.Contains(desc, "top-level intent_id") { + t.Fatalf("record_fact description must describe the facts-only contract: %q", desc) + } + // The seed the catalog stores is the same contract. + for _, seed := range BuiltinToolSeeds() { + if seed.Key != "record_fact" { + continue + } + if seed.Desc != desc { + t.Fatal("seed description differs from the tool") + } + if p := seed.Schema["properties"].(map[string]any); len(p) != 1 || p["facts"] == nil { + t.Fatalf("seed schema is not facts-only: %v", p) + } + } +} + +// A facts[] item without intent_id is linked to the intent the worker runs +// (SetOwnerNode); a legacy top-level intent_id still wins; a planner ToolSet +// (owner = the task root, not an intent) writes an unlinked fact. +func TestRecordFactDefaultsToRunningIntent(t *testing.T) { + store := guardTaskStore(t) + running := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "running intent"}, "running") + other := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "other intent"}, "open") + tools := NewToolSet(store, "worker") + tools.SetOwnerNode(running) + ids := func(res actool.Result) []int64 { + t.Helper() + var out struct { + IDs []int64 `json:"ids"` + } + if err := json.Unmarshal([]byte(res.Flatten()), &out); err != nil || len(out.IDs) != 1 || out.IDs[0] <= 0 { + t.Fatalf("batch result: %s", res.Flatten()) + } + return out.IDs + } + res := callGuardTool(t, tools.recordFact(), `{"facts":[{"summary":"linked to the running intent","confidence":"observed","source":"tool_output"}]}`) + if res.IsError { + t.Fatal(res.Flatten()) + } + first := ids(res)[0] + if yielded, err := store.FactsYielded(running); err != nil || len(yielded) != 1 || yielded[0] != first { + t.Fatalf("fact %d not linked to the running intent %d: %v %v", first, running, yielded, err) + } + res = callGuardTool(t, tools.recordFact(), `{"intent_id":`+strconv.FormatInt(other, 10)+`,"facts":[{"summary":"legacy top-level intent"}]}`) + if res.IsError { + t.Fatal(res.Flatten()) + } + second := ids(res)[0] + if yielded, err := store.FactsYielded(other); err != nil || len(yielded) != 1 || yielded[0] != second { + t.Fatalf("legacy top-level intent_id must win: %v %v", yielded, err) + } + res = callGuardTool(t, tools.recordFact(), `{"facts":[{"summary":"item intent","intent_id":`+strconv.FormatInt(other, 10)+`}]}`) + if res.IsError { + t.Fatal(res.Flatten()) + } + third := ids(res)[0] + if yielded, _ := store.FactsYielded(other); len(yielded) != 2 || yielded[1] != third && yielded[0] != third { + t.Fatalf("item intent_id must win: %v", yielded) + } + if yielded, _ := store.FactsYielded(running); len(yielded) != 1 { + t.Fatalf("running intent gained facts it did not produce: %v", yielded) + } + planner := NewToolSet(store, "planner") + origin, err := store.ListByKind(db.KindFact, 1) + if err != nil || len(origin) == 0 { + t.Fatalf("origin fact: %v", err) + } + planner.SetOwnerNode(origin[0].ID) + res = callGuardTool(t, planner.recordFact(), `{"facts":[{"summary":"planner fact without intent"}]}`) + if res.IsError || planner.Writes().Facts != 1 { + t.Fatalf("planner write refused: %s", res.Flatten()) + } + if yielded, _ := store.FactsYielded(running); len(yielded) != 1 { + t.Fatalf("planner fact must not be linked to a worker intent: %v", yielded) + } +} diff --git a/agent/record_fact_shapes_test.go b/agent/record_fact_shapes_test.go new file mode 100644 index 000000000..98835a3d3 --- /dev/null +++ b/agent/record_fact_shapes_test.go @@ -0,0 +1,128 @@ +package agent + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "testing" + + "github.com/Autumn-27/artex/db" +) + +// Synthetic replay of the record_fact call shapes a model produces, replayed +// against the handler in the package test database. The shapes mirror what +// was observed in production before the facts-only contract (a model filling +// every declared top-level property beside facts[]) without using recorded +// data. Checked per call: the verdict, that an accepted call writes exactly one +// node per facts item linked to the intent and anchored to the assets, and +// that a refused call writes nothing and names the top-level fields. +func TestRecordFactShapesReplay(t *testing.T) { + d := testDB(t) + t.Cleanup(func() { _ = d.Close() }) + task, err := d.CreateTaskWithOptions("record_fact shapes", "replay synthetic record_fact call shapes", db.TaskCreateOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = d.DeleteTask(task.ID) }) + store := d.Exploration(task.ExplorationID) + intentID := addGuardNode(t, store, db.KindIntent, map[string]any{"summary": "enumerate the service"}, "running") + var assets []int64 + for i := 1; i <= 2; i++ { + id, err := d.Assets().UpsertIP(db.UpsertIPReq{IP: fmt.Sprintf("203.0.113.%d", i), TaskID: task.ID}) + if err != nil { + t.Fatal(err) + } + assets = append(assets, id) + } + t.Cleanup(func() { _, _ = d.Assets().DeleteByIDs(assets) }) + tools := NewToolSet(store, "worker") + tools.SetOwnerNode(intentID) + tool := tools.recordFact() + + iid := strconv.FormatInt(intentID, 10) + a1, a2 := strconv.FormatInt(assets[0], 10), strconv.FormatInt(assets[1], 10) + item := func(n int) string { + return fmt.Sprintf(`{"summary":"conclusion %d","detail":"detail %d","evidence":"cmd %d → line","confidence":"observed","source":"tool_output","asset_ids":[%s]}`, n, n, n, a1) + } + type shape struct { + name string + call string + accept bool + items int + fields string // named in the refusal + } + shapes := []shape{ + {"published: one item", `{"facts":[` + item(1) + `]}`, true, 1, ""}, + {"published: two items", `{"facts":[` + item(1) + `,` + item(2) + `]}`, true, 2, ""}, + {"published: item with its own intent_id and alias", `{"facts":[{"summary":"alias","tested_asset_ids":["` + a2 + `"],"intent_id":` + iid + `}]}`, true, 1, ""}, + {"legacy batch: top-level intent_id", `{"intent_id":` + iid + `,"facts":[` + item(3) + `]}`, true, 1, ""}, + {"legacy single", `{"summary":"single fact","confidence":"inferred","asset_ids":[` + a2 + `]}`, true, 1, ""}, + {"copies of every field beside facts", `{"summary":"conclusion 1","detail":"detail 1","evidence":"cmd 1 → line","confidence":"observed","source":"tool_output","asset_ids":[` + a1 + `],"facts":[` + item(1) + `]}`, false, 1, `"asset_ids", "confidence", "detail", "evidence", "source", "summary"`}, + {"empty placeholders beside facts", `{"summary":"","detail":"","asset_ids":[],"tested_asset_ids":[],"facts":[` + item(1) + `]}`, false, 1, `"asset_ids", "detail", "summary", "tested_asset_ids"`}, + {"dummy summary beside facts", `{"summary":"x","facts":[` + item(1) + `]}`, false, 1, `"summary"`}, + {"abbreviated copy beside facts", `{"summary":"short","detail":"short detail","facts":[` + item(1) + `]}`, false, 1, `"detail", "summary"`}, + {"conflicting source beside facts", `{"source":"inference","facts":[` + item(1) + `]}`, false, 1, `"source"`}, + {"top-level summary beside two items", `{"summary":"conclusion 1","facts":[` + item(1) + `,` + item(2) + `]}`, false, 2, `"summary"`}, + } + for _, s := range shapes { + before := tools.Writes().Facts + linked, _ := store.FactsYielded(intentID) + res := callGuardTool(t, tool, s.call) + written := tools.Writes().Facts - before + if !res.IsError != s.accept { + t.Errorf("%s: accepted=%v want %v: %s", s.name, !res.IsError, s.accept, res.Flatten()) + continue + } + if !s.accept { + if written != 0 || !strings.Contains(res.Flatten(), "top-level field(s) "+s.fields) || !strings.Contains(res.Flatten(), "no facts were written") { + t.Errorf("%s: refusal must write nothing and name %s: %s (writes %d)", s.name, s.fields, res.Flatten(), written) + } + continue + } + if written != s.items { + t.Errorf("%s: wrote %d facts for %d items", s.name, written, s.items) + } + after, _ := store.FactsYielded(intentID) + if len(after) != len(linked)+s.items { + t.Errorf("%s: intent links %d → %d for %d items", s.name, len(linked), len(after), s.items) + } + ids := after[len(linked):] + if strings.HasPrefix(s.call, `{"facts"`) || strings.HasPrefix(s.call, `{"intent_id"`) { + var out struct { + IDs []int64 `json:"ids"` + } + if err := json.Unmarshal([]byte(res.Flatten()), &out); err != nil || len(out.IDs) != s.items { + t.Errorf("%s: ids: %s", s.name, res.Flatten()) + } + } + for _, id := range ids { + node, err := store.GetNode(id) + if err != nil || node == nil || node.Kind != db.KindFact { + t.Fatalf("%s: fact %d: %v", s.name, id, err) + } + var payload map[string]any + _ = json.Unmarshal(node.Payload, &payload) + if sum, _ := payload["summary"].(string); sum == "" || sum == "x" { + t.Errorf("%s: stored summary %q", s.name, sum) + } + if strings.Contains(s.call, "asset_ids") { + anchored := false + for _, asset := range assets { + refs, err := store.AssetRefs(asset) + if err != nil { + t.Fatal(err) + } + for _, r := range refs { + if r.ID == id { + anchored = true + } + } + } + if !anchored { + t.Errorf("%s: fact %d not anchored to an asset", s.name, id) + } + } + } + } +} diff --git a/agent/toolcatalog.go b/agent/toolcatalog.go index 5811203b5..6e85c7eb4 100644 --- a/agent/toolcatalog.go +++ b/agent/toolcatalog.go @@ -180,6 +180,14 @@ type overriddenTool struct { func (o *overriddenTool) Description() string { return o.desc } func (o *overriddenTool) InputSchema() map[string]any { return o.schema } +// AcceptsRawInput forwards the wrapped tool's opt-out of the harness schema +// check (record_fact validates its own input); the embedded interface alone +// would hide it from the harness's type assertion. +func (o *overriddenTool) AcceptsRawInput() bool { + r, ok := o.CoreTool.(interface{ AcceptsRawInput() bool }) + return ok && r.AcceptsRawInput() +} + func (o *overriddenTool) Call(ctx context.Context, in json.RawMessage, tc *actool.ToolContext) (actool.Result, error) { return o.CoreTool.Call(ctx, injectDefaults(in, o.schema), tc) } diff --git a/agent/tools.go b/agent/tools.go index a34f0e1dd..939871cce 100644 --- a/agent/tools.go +++ b/agent/tools.go @@ -394,8 +394,21 @@ func readTool(name, desc string, schema map[string]any, run func(context.Context } func writeTool(name, desc string, schema map[string]any, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool { + return buildWriteTool(name, desc, schema, false, run) +} + +// writeRawTool is writeTool for a handler that validates its own input: the +// harness skips its schema check (Spec.RawInput), so the published schema can +// be stricter (required properties) than the legacy shapes the handler still +// accepts, and refusals carry the handler's own message instead of the +// SDK's generic one. +func writeRawTool(name, desc string, schema map[string]any, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool { + return buildWriteTool(name, desc, schema, true, run) +} + +func buildWriteTool(name, desc string, schema map[string]any, raw bool, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool { return actool.Build(actool.Spec{ - Name: name, Description: desc, Schema: schema, + Name: name, Description: desc, Schema: schema, RawInput: raw, Permissions: func(context.Context, json.RawMessage, acperm.Context) acperm.Decision { return acperm.Allowed() }, Run: func(ctx context.Context, in json.RawMessage, _ *actool.ToolContext) (actool.Result, error) { return run(ctx, in) @@ -419,6 +432,12 @@ func (t *ToolSet) writeExpTool(name, desc string, schema map[string]any, run fun return writeTool(name, desc, schema, t.needExploration(name, run)) } +// writeExpToolRaw is writeExpTool with the harness schema check skipped (see +// writeRawTool); the handler validates the input itself. +func (t *ToolSet) writeExpToolRaw(name, desc string, schema map[string]any, run func(context.Context, json.RawMessage) (actool.Result, error)) actool.CoreTool { + return writeRawTool(name, desc, schema, t.needExploration(name, run)) +} + // needExploration wraps a handler so it only runs with an exploration store. // Tools that degrade more usefully than "unavailable" (report_finding points at // add_task_hint, set_goals/set_constraints at the task itself) keep their own @@ -1588,15 +1607,30 @@ func (t *ToolSet) recordOneFact(it factItem, defaultIntent int64) (int64, error) return id, nil } +// runningIntent is the intent this ToolSet works for (the worker's claimed +// intent, set by SetOwnerNode), or 0 when the owner node is not an intent +// (planner: the task root) or unknown. +func (t *ToolSet) runningIntent() int64 { + if t.ts == nil || t.ownerNode <= 0 { + return 0 + } + node, err := t.ts.GetNode(t.ownerNode) + if err != nil || node == nil || node.Kind != db.KindIntent { + return 0 + } + return node.ID +} + func (t *ToolSet) recordFact() actool.CoreTool { - return t.writeExpTool("record_fact", "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n"+ + return t.writeExpToolRaw("record_fact", "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n"+ "⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n"+ - "★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n"+ + "★facts is the only parameter: facts:[{...}] with one item per [mutually distinct] conclusion, usually exactly one item. Returns an ids array, same length and order as facts. An item without intent_id is linked to the intent you are running.\n"+ "⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n"+ " · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n"+ " · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n"+ " · source = where the content came from: target_response (content the target itself returned: page bodies, headers, error messages, banners, even when a tool fetched it) | tool_output (a tool's own result about the target: port states, scanner verdicts, status codes, timings) | third_party_intel (OSINT, CVE databases, public reports) | operator_input (given by the human operator) | inference (your own reasoning; the default when omitted). Any other value is refused.\n"+ - " · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen.", + " · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen."+ + recordFactContractNote, recordFactSchema(), func(_ context.Context, in json.RawMessage) (actool.Result, error) { a, batch, err := decodeRecordFactInput(in) @@ -1607,7 +1641,12 @@ func (t *ToolSet) recordFact() actool.CoreTool { if !batch { items = []factItem{a.factItem} } - defaultIntent := pid(a.factItem.IntentID) // top-level intent_id = batch default + // Default intent of the items: a legacy top-level intent_id, else the + // intent this worker is running (the published schema has no top level). + defaultIntent := pid(a.factItem.IntentID) + if defaultIntent <= 0 { + defaultIntent = t.runningIntent() + } ids := make([]int64, len(items)) errs := map[string]string{} diff --git a/server/fact_input_released_test.go b/server/fact_input_released_test.go new file mode 100644 index 000000000..4d2116e69 --- /dev/null +++ b/server/fact_input_released_test.go @@ -0,0 +1,117 @@ +package server + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "testing" +) + +// The record_fact texts seeded by the releases v0.4.0 to v0.6.0, copied from +// agent/tools.go at those tags. They are kept here, apart from the agent +// package's list of known defaults, so the test fails when that list drifts +// from the released bytes. +const fix410ReleasedDescription = "Write an exploration [fact/conclusion] into the exploration graph, linked to the intent that produced it (intent_id). Used to record exploration results — both [positive conclusions] like fingerprinting/enumeration, and [negative conclusions] like 'port closed'/'parameter not injectable'/'no login entry found'.\n" + + "⚠️Multiple observations from one exploration must be [consolidated into ONE fact], not split into many; whenever they can be merged into one fact, use one: summary = a one-sentence summary of this conclusion, detail = the related details (may contain multiple concrete items). Example: a fingerprinting intent → one fact {summary:'identified the tech stack and response characteristics of site X', detail:'nginx 1.25 / Vue3 / 200 / title=.. / body_len=..'}, rather than one fact each for status code, fingerprint, and title. One intent usually produces exactly one fact; splitting too finely bloats the graph endlessly.\n" + + "★The facts array writes several [mutually distinct] conclusions at once (each may omit intent_id, defaulting to the top-level intent_id). Returns an ids array, same length and order as facts.\n" + + "⚠️Write only conclusions you [actually saw] in tool output — don't extrapolate. evidence and confidence keep inaccurate conclusions from polluting the graph:\n" + + " · evidence = [one line] of key evidence supporting this conclusion (the command + the one or two most proving output lines), **keep it brief** — details belong in detail; don't paste large output blocks here.\n" + + " · confidence = observed (seen directly in output) | inferred (inferred from phenomena).\n" + + " · **Negative conclusions** (not injectable / port closed / no entry found, etc.) state only \"observation + tentative reading\" — state what you actually saw; whether to abandon the direction is for the planner to decide globally; always provide evidence, mark inferred when the means weren't exhausted or the evidence is weak (including probed only once, looks-like), and mark observed only when truly exhausted and directly seen." + +const fix410ReleasedFactsDescription = "[use when there are several distinct conclusions] array of facts; element fields same as the top-level fields below (summary/detail/evidence/confidence/intent_id/asset_ids); omitting intent_id falls back to the top-level intent_id. The returned ids match this array in length and order." + +// A database seeded by a release stores the released record_fact description, +// the released facts property description and facts items without properties. +// The one-time refresh replaces both texts with the current ones and sets its +// flag; texts an operator edited match no known default and stay as stored. +func TestFix410ReleasedDefaultsRefreshed(t *testing.T) { + for _, c := range []struct { + name, text, sum string + size int + }{ + {"description", fix410ReleasedDescription, "0da5cec5cde7f5e6f41343418fcc2a7d132d6520f336efb39fcbf1ab6ac51957", 2038}, + {"facts description", fix410ReleasedFactsDescription, "eb054843a6b33dd39b94ba0648f292babb9cf9bdd106ee4eff6bf593d3f3edf4", 286}, + } { + sum := sha256.Sum256([]byte(c.text)) + if len(c.text) != c.size || hex.EncodeToString(sum[:]) != c.sum { + t.Fatalf("the released %s in this test is not the released text: %d bytes, sha256 %x", c.name, len(c.text), sum) + } + } + skipWithoutDSN(t) + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + defer m.Close() + s := &Server{m: m} + const flag = "tool_schema_refresh_record_fact_facts_only" + seed := recordFactSeed(t) + seedSchema, _ := json.Marshal(seed.Schema) + seedAgents, _ := json.Marshal(seed.Agents) + defer func() { + _ = m.pg.UpsertToolForce("record_fact", seed.Desc, seedSchema, seedAgents) + _, _ = m.pg.Exec(`DELETE FROM settings WHERE key=$1`, flag) + }() + currentFacts := seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)["description"] + + // start stores the row as a release seeded it, with the given two texts, + // runs the startup steps and returns the two texts as stored afterwards. + start := func(desc, factsDesc string) (string, any) { + t.Helper() + str := func(d string) map[string]any { return map[string]any{"type": "string", "description": d} } + schema, _ := json.Marshal(map[string]any{"type": "object", "properties": map[string]any{ + "facts": map[string]any{"type": "array", "description": factsDesc, "items": map[string]any{"type": "object"}}, + "summary": str("a [one-sentence summary] of this exploration's conclusion (a condensation of detail)"), + "intent_id": map[string]any{"type": "integer", "description": "the id of the intent that produced this fact (the intent you claimed; the default for each item in batch mode)"}, + "detail": str("details related to this fact: write all of this exploration's observations here"), + "evidence": str("[one line] of key evidence: the command + the one or two most proving output lines. Keep it brief; don't paste large output blocks (details go in detail)."), + "confidence": str("observed (seen directly in output) | inferred (inferred from phenomena). Always label negative conclusions honestly."), + "asset_ids": map[string]any{"type": "array", "items": map[string]any{"type": "integer"}, "description": "related asset ids (optional, 0/1/many): which assets this fact involves"}, + }}) + if err := m.pg.UpsertToolForce("record_fact", desc, schema, seedAgents); err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec(`DELETE FROM settings WHERE key=$1`, flag); err != nil { + t.Fatal(err) + } + s.seedFactInputSchema() + s.seedFactSourceSchema() + s.refreshRecordFactContract() + if v, _, _ := m.pg.GetSetting(flag); v != "true" { + t.Errorf("flag = %q after the refresh, want true", v) + } + row, err := m.pg.GetTool("record_fact") + if err != nil || row == nil { + t.Fatalf("row: %v", err) + } + var stored struct { + Properties map[string]map[string]any `json:"properties"` + } + if err := json.Unmarshal(row.Schema, &stored); err != nil { + t.Fatal(err) + } + if len(stored.Properties) != 1 || stored.Properties["facts"] == nil { + t.Fatalf("top level still holds fact fields: %s", row.Schema) + } + return row.Description, stored.Properties["facts"]["description"] + } + + desc, factsDesc := start(fix410ReleasedDescription, fix410ReleasedFactsDescription) + if desc != seed.Desc { + t.Errorf("released description not replaced by the current one: %.60q", desc) + } + if factsDesc != currentFacts { + t.Errorf("released facts description not replaced by the current one: %v", factsDesc) + } + + editedDesc := fix410ReleasedDescription + "\nOperator note: record every negative result." + editedFacts := fix410ReleasedFactsDescription + " Operator note: one fact per host." + desc, factsDesc = start(editedDesc, editedFacts) + if desc != editedDesc { + t.Errorf("operator description changed: %.60q", desc) + } + if factsDesc != editedFacts { + t.Errorf("operator facts description changed: %v", factsDesc) + } +} diff --git a/server/fact_input_schema.go b/server/fact_input_schema.go index 22ffed682..31ad152e0 100644 --- a/server/fact_input_schema.go +++ b/server/fact_input_schema.go @@ -9,6 +9,99 @@ import ( // SeedTool only inserts new rows. Repair existing built-ins too, while keeping // descriptions, bindings, enabled/deferred flags and parameter annotations. + +// refreshRecordFactContract brings an already-seeded record_fact row to the +// facts-only contract (one-time, settings flag). SeedTool is first-insert-only +// and ToolResolve serves the stored text, so without this an old database keeps +// describing the earlier contract (fact fields at the top level beside facts[]) +// to the model. Only text equal to the previous code default is replaced: +// - the tool description; +// - the facts property and item descriptions that referred to the top level. +// +// Operator edits stay; a description that matches no known default is logged +// as such and left in place. The schema structure itself is repaired by +// seedFactInputSchema on every start (RepairRecordFactSchema). +// +// The flag is written only once the outcome is known: the UPDATE succeeded +// (also when it changed no row) or the row is not the built-in one. A failed +// read, decode or UPDATE leaves it unset, so the next start retries. +func (s *Server) refreshRecordFactContract() { + const flag = "tool_schema_refresh_record_fact_facts_only" + if v, _, _ := s.m.pg.GetSetting(flag); v == "true" { + return + } + done := func() { _ = s.m.pg.SetSetting(flag, "true") } + var seed *agent.ToolSeed + for _, sd := range agent.BuiltinToolSeeds() { + if sd.Key == "record_fact" { + seed = &sd + break + } + } + row, err := s.m.pg.GetTool("record_fact") + if err != nil { + log.Printf("[facts] read record_fact for the contract refresh: %v", err) + return + } + if row == nil || seed == nil { + return + } + if !row.System || row.Kind != "builtin" { + done() + return + } + legacyTool, legacyFacts, legacyItems := agent.RecordFactLegacyDescriptions() + desc := row.Description + switch { + case desc == seed.Desc: + case sliceHas(legacyTool, desc): + desc = seed.Desc + default: + log.Printf("[facts] record_fact description does not match a known default (operator edit or an older release); the facts-only schema is published anyway, review the text in the tool catalog") + } + var schema map[string]any + if err := json.Unmarshal(row.Schema, &schema); err != nil { + log.Printf("[facts] decode record_fact schema: %v", err) + return + } + repaired := agent.RepairRecordFactSchema(schema) + facts := repaired["properties"].(map[string]any)["facts"].(map[string]any) + canonFacts := seed.Schema["properties"].(map[string]any)["facts"].(map[string]any) + if d, ok := facts["description"].(string); ok && sliceHas(legacyFacts, d) { + facts["description"] = canonFacts["description"] + } + items := facts["items"].(map[string]any)["properties"].(map[string]any) + canonItems := canonFacts["items"].(map[string]any)["properties"].(map[string]any) + for name, old := range legacyItems { + prop, _ := items[name].(map[string]any) + want, _ := canonItems[name].(map[string]any) + if prop != nil && want != nil && prop["description"] == old { + prop["description"] = want["description"] + } + } + raw, err := json.Marshal(repaired) + if err != nil { + return + } + if _, err := s.m.pg.Exec(`UPDATE tools SET description=$1, schema=$2::jsonb, updated_at=now() +WHERE key='record_fact' AND system AND kind='builtin' + AND (description IS DISTINCT FROM $1 OR schema IS DISTINCT FROM $2::jsonb)`, desc, string(raw)); err != nil { + log.Printf("[facts] refresh record_fact contract: %v", err) + return + } + done() + log.Printf("[facts] record_fact now publishes the facts-only contract (one-time)") +} + +func sliceHas(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + func (s *Server) seedFactInputSchema() { row, err := s.m.pg.GetTool("record_fact") if err != nil { diff --git a/server/fact_input_schema_test.go b/server/fact_input_schema_test.go index 4114633a6..959697147 100644 --- a/server/fact_input_schema_test.go +++ b/server/fact_input_schema_test.go @@ -3,6 +3,7 @@ package server import ( "context" "encoding/json" + "strings" "testing" "time" @@ -42,10 +43,16 @@ func TestRecordFactInputSchemaUpgrade(t *testing.T) { if err := pg.UpdateTool("record_fact", "operator tool", raw, bindings, old.Enabled); err != nil { t.Fatal(err) } + // Facts-only contract: the top level holds facts and + // nothing else; the earlier top-level fact fields and their operator + // annotations are dropped (the handler still accepts the legacy shapes), + // item annotations survive. check := func(schema map[string]any) { t.Helper() - if _, ok := schema["required"]; ok { - t.Fatal("top-level summary still required for batches") + if req, _ := schema["required"].([]any); len(req) != 1 || req[0] != "facts" { + if reqS, _ := schema["required"].([]string); len(reqS) != 1 || reqS[0] != "facts" { + t.Fatalf("facts must be the only required property: %v", schema["required"]) + } } if _, ok := schema["anyOf"]; ok { t.Fatal("root anyOf still present") @@ -54,14 +61,13 @@ func TestRecordFactInputSchemaUpgrade(t *testing.T) { t.Fatal("unknown fields allowed") } props := schema["properties"].(map[string]any) + if len(props) != 1 || props["facts"] == nil { + t.Fatalf("top level must hold facts only: %v", props) + } items := props["facts"].(map[string]any)["items"].(map[string]any) itemProps := items["properties"].(map[string]any) - if props["tested_asset_ids"] == nil || itemProps["tested_asset_ids"] == nil || itemProps["summary"] == nil { - t.Fatal("repaired schema missing alias or batch summary") - } - if props["summary"].(map[string]any)["description"] != "operator summary" || - props["confidence"].(map[string]any)["default"] != "inferred" || props["custom"] != nil { - t.Fatal("operator annotations lost") + if itemProps["tested_asset_ids"] == nil || itemProps["summary"] == nil || itemProps["source"] == nil { + t.Fatal("repaired schema missing alias, batch summary or source") } if err := actool.ValidateInput(schema, json.RawMessage(`{"facts":[{"summary":"complete batch item"}]}`)); err != nil { t.Fatal("valid batch rejected", err) @@ -104,3 +110,217 @@ func TestRecordFactInputSchemaUpgrade(t *testing.T) { t.Fatal("second migration changed the row") } } + +// recordFactLegacySchema is the stored record_fact schema of the earlier +// contract: fact fields at the top level beside facts[], with the texts that +// referred to the top level, plus an operator annotation on an item field. +func recordFactLegacySchema(t *testing.T) (string, map[string]string) { + t.Helper() + _, legacyFacts, legacyItems := agent.RecordFactLegacyDescriptions() + item := map[string]any{"type": "object", "required": []string{"summary"}, "additionalProperties": false, "properties": map[string]any{ + "summary": map[string]any{"type": "string", "minLength": 1, "description": legacyItems["summary"]}, + "intent_id": map[string]any{"type": "integer", "description": legacyItems["intent_id"]}, + "detail": map[string]any{"type": "string", "description": "Details of this fact, including all related observations."}, + "confidence": map[string]any{"type": "string", "description": "operator confidence note", "default": "inferred"}, + "source": map[string]any{"type": "string", "enum": []string{"target_response", "tool_output", "third_party_intel", "operator_input", "inference"}, "description": "where the content came from"}, + }} + schema := map[string]any{"type": "object", "additionalProperties": false, "properties": map[string]any{ + "summary": map[string]any{"type": "string", "minLength": 1, "description": legacyItems["summary"]}, + "intent_id": map[string]any{"type": "integer", "description": legacyItems["intent_id"]}, + "detail": map[string]any{"type": "string", "description": "Details of this fact, including all related observations."}, + "source": map[string]any{"type": "string", "enum": []string{"inference"}}, + "facts": map[string]any{"type": "array", "minItems": 1, "description": legacyFacts[0], "items": item}, + }} + raw, _ := json.Marshal(schema) + return string(raw), legacyItems +} + +func recordFactSeed(t *testing.T) agent.ToolSeed { + t.Helper() + for _, sd := range agent.BuiltinToolSeeds() { + if sd.Key == "record_fact" { + return sd + } + } + t.Fatal("record_fact seed missing") + return agent.ToolSeed{} +} + +// A database that still stores the earlier record_fact contract (text and +// schema) is brought to the facts-only contract once: description replaced +// when it is the previous default, facts/item descriptions replaced when they +// are the earlier texts, operator annotations kept, top-level fact fields +// gone, source enum present on the items. A second run changes nothing, and +// an operator-edited description is left alone. +func TestRefreshRecordFactContract(t *testing.T) { + skipWithoutDSN(t) + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Skipf("postgres unavailable (%v)", err) + } + defer m.Close() + s := &Server{m: m} + seed := recordFactSeed(t) + seedSchema, _ := json.Marshal(seed.Schema) + seedAgents, _ := json.Marshal(seed.Agents) + restore := func() { + _ = m.pg.UpsertToolForce("record_fact", seed.Desc, seedSchema, seedAgents) + _, _ = m.pg.Exec(`DELETE FROM settings WHERE key='tool_schema_refresh_record_fact_facts_only'`) + } + defer restore() + + legacyTool, _, legacyItems := agent.RecordFactLegacyDescriptions() + legacySchema, _ := recordFactLegacySchema(t) + run := func(desc string) { + t.Helper() + if err := m.pg.UpsertToolForce("record_fact", desc, json.RawMessage(legacySchema), seedAgents); err != nil { + t.Fatal(err) + } + if _, err := m.pg.Exec(`DELETE FROM settings WHERE key='tool_schema_refresh_record_fact_facts_only'`); err != nil { + t.Fatal(err) + } + s.seedFactInputSchema() + s.seedFactSourceSchema() + s.refreshRecordFactContract() + } + for _, desc := range legacyTool { + run(desc) + row, err := m.pg.GetTool("record_fact") + if err != nil || row == nil { + t.Fatalf("row: %v", err) + } + if row.Description != seed.Desc { + t.Fatalf("description not refreshed from a known default: %q", row.Description[:60]) + } + var schema map[string]any + if err := json.Unmarshal(row.Schema, &schema); err != nil { + t.Fatal(err) + } + props := schema["properties"].(map[string]any) + if len(props) != 1 || props["facts"] == nil { + t.Fatalf("top level still holds fact fields: %v", props) + } + facts := props["facts"].(map[string]any) + items := facts["items"].(map[string]any)["properties"].(map[string]any) + canonItems := seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any) + if facts["description"] != seed.Schema["properties"].(map[string]any)["facts"].(map[string]any)["description"] { + t.Fatalf("facts description not refreshed: %v", facts["description"]) + } + for name := range legacyItems { + if items[name].(map[string]any)["description"] != canonItems[name].(map[string]any)["description"] { + t.Fatalf("item %s description not refreshed: %v", name, items[name]) + } + } + if items["confidence"].(map[string]any)["description"] != "operator confidence note" || items["confidence"].(map[string]any)["default"] != "inferred" { + t.Fatalf("operator annotation lost: %v", items["confidence"]) + } + if enum, _ := items["source"].(map[string]any)["enum"].([]any); len(enum) != 5 { + t.Fatalf("source enum missing on the items: %v", items["source"]) + } + if v, _, _ := m.pg.GetSetting("tool_schema_refresh_record_fact_facts_only"); v != "true" { + t.Fatal("flag not set") + } + // Second start: nothing changes. + var updated1, updated2 string + _ = m.pg.QueryRow(`SELECT updated_at::text FROM tools WHERE key='record_fact'`).Scan(&updated1) + s.seedFactInputSchema() + s.seedFactSourceSchema() + s.refreshRecordFactContract() + _ = m.pg.QueryRow(`SELECT updated_at::text FROM tools WHERE key='record_fact'`).Scan(&updated2) + if updated1 != updated2 { + t.Fatal("second start rewrote the row") + } + } + // An operator-edited description is kept; the schema is still repaired. + run("my own record_fact text") + row, _ := m.pg.GetTool("record_fact") + if row == nil || row.Description != "my own record_fact text" { + t.Fatalf("operator description replaced: %v", row) + } + if !strings.Contains(string(row.Schema), `"facts"`) || strings.Contains(string(row.Schema), legacyItems["intent_id"]) { + t.Fatalf("schema not repaired beside an operator description: %s", row.Schema) + } +} + +// A start on which the refresh fails must not spend the one-time flag: the +// next start retries and the stored default text is refreshed then. +func TestFix410RefreshFlagNotSetOnFailure(t *testing.T) { + skipWithoutDSN(t) + m, err := NewManager(t.TempDir(), "") + if err != nil { + t.Skipf("postgres unavailable (%v)", err) + } + defer m.Close() + s := &Server{m: m} + const flag = "tool_schema_refresh_record_fact_facts_only" + seed := recordFactSeed(t) + seedSchema, _ := json.Marshal(seed.Schema) + seedAgents, _ := json.Marshal(seed.Agents) + legacyTool, _, _ := agent.RecordFactLegacyDescriptions() + legacySchema, _ := recordFactLegacySchema(t) + exec := func(query string, args ...any) { + t.Helper() + if _, err := m.pg.Exec(query, args...); err != nil { + t.Fatal(err) + } + } + const dropTrigger = `DROP TRIGGER IF EXISTS fix410_fail_update ON tools` + defer func() { + _, _ = m.pg.Exec(dropTrigger) + _, _ = m.pg.Exec(`DROP FUNCTION IF EXISTS fix410_fail_update()`) + _ = m.pg.UpsertToolForce("record_fact", seed.Desc, seedSchema, seedAgents) + _, _ = m.pg.Exec(`DELETE FROM settings WHERE key=$1`, flag) + }() + exec(`CREATE OR REPLACE FUNCTION fix410_fail_update() RETURNS trigger LANGUAGE plpgsql AS +'BEGIN RAISE EXCEPTION ''fix410: simulated transient failure''; END'`) + + start := func() { + s.seedFactInputSchema() + s.seedFactSourceSchema() + s.refreshRecordFactContract() + } + for _, tc := range []struct { + name string + fail, clear func() + }{ + {"update fails", func() { + exec(`CREATE TRIGGER fix410_fail_update BEFORE UPDATE ON tools +FOR EACH ROW WHEN (OLD.key = 'record_fact') EXECUTE FUNCTION fix410_fail_update()`) + }, func() { exec(dropTrigger) }}, + {"schema does not decode", func() { + exec(`UPDATE tools SET schema='[]'::jsonb WHERE key='record_fact'`) + }, func() { + exec(`UPDATE tools SET schema=$1::jsonb WHERE key='record_fact'`, legacySchema) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + // A database that stores the earlier default text and has not run the refresh. + if err := m.pg.UpsertToolForce("record_fact", legacyTool[0], json.RawMessage(legacySchema), seedAgents); err != nil { + t.Fatal(err) + } + exec(`DELETE FROM settings WHERE key=$1`, flag) + + tc.fail() + start() + if v, ok, _ := m.pg.GetSetting(flag); ok { + t.Errorf("flag is %q after a failed refresh: later starts skip it", v) + } + if row, _ := m.pg.GetTool("record_fact"); row == nil || row.Description != legacyTool[0] { + t.Fatal("the simulated failure did not leave the earlier description in place") + } + + tc.clear() + start() + row, err := m.pg.GetTool("record_fact") + if err != nil || row == nil { + t.Fatalf("row: %v", err) + } + if row.Description != seed.Desc { + t.Errorf("the next start did not refresh the description: %.60q", row.Description) + } + if v, _, _ := m.pg.GetSetting(flag); v != "true" { + t.Errorf("flag = %q after a successful refresh, want true", v) + } + }) + } +} diff --git a/server/fact_source_schema_test.go b/server/fact_source_schema_test.go index a087b4fc0..baa30931a 100644 --- a/server/fact_source_schema_test.go +++ b/server/fact_source_schema_test.go @@ -35,12 +35,13 @@ func TestRecordFactSourceSchemaUpgrade(t *testing.T) { t.Cleanup(func() { pg.UpdateTool(old.Key, old.Description, old.Schema, bindings, old.Enabled) }) // Simulate a pre-provenance row customised by the operator: no source - // property, an edited description and an edited parameter description. + // property on the facts items, an edited description and an edited + // parameter description (facts-only contract: the items carry the fields). var schema map[string]any if err := json.Unmarshal(old.Schema, &schema); err != nil { t.Fatal(err) } - props := schema["properties"].(map[string]any) + props := schema["properties"].(map[string]any)["facts"].(map[string]any)["items"].(map[string]any)["properties"].(map[string]any) delete(props, "source") props["confidence"].(map[string]any)["description"] = "operator confidence text" custom, _ := json.Marshal(schema) @@ -72,16 +73,23 @@ func TestRecordFactSourceSchemaUpgrade(t *testing.T) { t.Fatal(err) } var got struct { - Properties map[string]map[string]any `json:"properties"` + Properties struct { + Facts struct { + Items struct { + Properties map[string]map[string]any `json:"properties"` + } `json:"items"` + } `json:"facts"` + } `json:"properties"` } if err := json.Unmarshal(tool.Schema, &got); err != nil { t.Fatal(err) } + itemProps := got.Properties.Facts.Items.Properties wantEnum := "[target_response tool_output third_party_intel operator_input inference]" - if src := got.Properties["source"]; src == nil || fmt.Sprint(src["enum"]) != wantEnum { + if src := itemProps["source"]; src == nil || fmt.Sprint(src["enum"]) != wantEnum { t.Fatalf("upgrade did not add the source enum: %s", tool.Schema) } - if tool.Description != "operator description" || got.Properties["confidence"]["description"] != "operator confidence text" || len(got.Properties) != len(props)+1 { + if tool.Description != "operator description" || itemProps["confidence"]["description"] != "operator confidence text" || len(itemProps) != len(props)+1 { t.Fatalf("operator configuration overwritten: %q %s", tool.Description, tool.Schema) } @@ -92,15 +100,18 @@ func TestRecordFactSourceSchemaUpgrade(t *testing.T) { } } -// resolvedRecordFactProps returns the properties of record_fact as ToolResolve -// serves it to the worker. +// resolvedRecordFactProps returns the facts[] item properties of record_fact as +// ToolResolve serves it to the worker. func resolvedRecordFactProps(t *testing.T) map[string]any { t.Helper() tools := agent.ToolResolve()(context.Background(), "worker", agent.NewToolSet(nil, "").WorkerTools()) for _, tl := range tools { if tl.Name() == "record_fact" { props, _ := tl.InputSchema()["properties"].(map[string]any) - return props + facts, _ := props["facts"].(map[string]any) + items, _ := facts["items"].(map[string]any) + itemProps, _ := items["properties"].(map[string]any) + return itemProps } } t.Fatal("worker lost record_fact") diff --git a/server/orchestration.go b/server/orchestration.go index bf88343e6..9b90deb75 100644 --- a/server/orchestration.go +++ b/server/orchestration.go @@ -573,6 +573,7 @@ func (s *Server) seedOrchestrationTools() { s.seedFindingEvidenceSchema() // optional counterevidence / confidence_rationale on report_finding, added once per DB s.seedFactInputSchema() // restore record_fact single/batch contracts s.seedFactSourceSchema() // source enum on record_fact, added once per DB + s.refreshRecordFactContract() // record_fact publishes the facts-only contract (one-time) s.seedTaskScopeDescription() // add_task_scope: "coverage scope, not enforced" replaces the authorization-boundary text // Note: pentest's default tool bindings need no migration — BuiltinToolSeeds already seeds // list_assets/insert_assets/report_finding/list_findings/list_companies together with @@ -602,11 +603,23 @@ func (s *Server) seedFactSourceSchema() { if err := json.Unmarshal(obj["properties"], &properties); err != nil { return } - if len(properties["source"]) == 0 { + // The source enum lives on the facts[] items (facts-only contract); the + // top level has no fact fields any more. + var facts map[string]json.RawMessage + var items map[string]json.RawMessage + var itemProps map[string]json.RawMessage + if err := json.Unmarshal(properties["facts"], &facts); err != nil { return } - _, err = s.m.pg.Exec(`UPDATE tools SET schema=jsonb_set(schema,ARRAY['properties','source'],$1::jsonb,true),updated_at=now() -WHERE key='record_fact' AND system AND NOT(COALESCE(schema->'properties','{}'::jsonb) ? 'source')`, string(properties["source"])) + if err := json.Unmarshal(facts["items"], &items); err != nil { + return + } + if err := json.Unmarshal(items["properties"], &itemProps); err != nil || len(itemProps["source"]) == 0 { + return + } + _, err = s.m.pg.Exec(`UPDATE tools SET schema=jsonb_set(schema,ARRAY['properties','facts','items','properties','source'],$1::jsonb,true),updated_at=now() +WHERE key='record_fact' AND system AND (schema->'properties'->'facts'->'items') ? 'properties' + AND NOT(COALESCE(schema->'properties'->'facts'->'items'->'properties','{}'::jsonb) ? 'source')`, string(itemProps["source"])) if err != nil { log.Printf("[facts] upgrade record_fact.source: %v", err) } diff --git a/server/record_fact_rawinput_test.go b/server/record_fact_rawinput_test.go new file mode 100644 index 000000000..92a643abd --- /dev/null +++ b/server/record_fact_rawinput_test.go @@ -0,0 +1,119 @@ +package server + +import ( + "context" + "strings" + "testing" + + "github.com/Autumn-27/norma/harness" + "github.com/Autumn-27/norma/llm" + actool "github.com/Autumn-27/norma/tool" + + "github.com/Autumn-27/artex/agent" + "github.com/Autumn-27/artex/db" +) + +// acceptsRawInput is the assertion norma's harness makes before its schema +// check: only a tool that reports AcceptsRawInput skips it. +func acceptsRawInput(t actool.CoreTool) bool { + r, ok := t.(interface{ AcceptsRawInput() bool }) + return ok && r.AcceptsRawInput() +} + +// record_fact opts out of the harness schema check and validates its own +// input. Every wrapper between the handler and the harness has to forward +// that opt-out: an embedded CoreTool alone hides it from the harness's type +// assertion, and the stricter published schema then rejects the calls the +// handler exists to answer. +func TestFix410AcceptsRawInputSurvivesWrappers(t *testing.T) { + oldAugment, oldResolve := agent.ToolAugment(), agent.ToolResolve() + t.Cleanup(func() { + agent.SetToolAugment(oldAugment) + agent.SetToolResolve(oldResolve) + }) + ctx := context.Background() + probe := func(raw bool) actool.CoreTool { + return actool.Build(actool.Spec{Name: "fix410_probe", Schema: map[string]any{"type": "object"}, RawInput: raw}) + } + + // AugmentTools without hooks applies guardPanic and nothing else. + t.Run("guardedTool", func(t *testing.T) { + agent.SetToolAugment(nil) + agent.SetToolResolve(nil) + for _, raw := range []bool{true, false} { + out, _, cleanup := agent.AugmentTools(ctx, "worker", []actool.CoreTool{probe(raw)}) + cleanup() + if len(out) != 1 { + t.Fatalf("AugmentTools returned %d tools, want 1", len(out)) + } + if got := acceptsRawInput(out[0]); got != raw { + t.Errorf("%T: AcceptsRawInput = %v, wrapped tool reports %v", out[0], got, raw) + } + } + }) + + t.Run("meteredTool", func(t *testing.T) { + for _, raw := range []bool{true, false} { + wrapped := meterTool(probe(raw), &recordingToolUsage{}, "fix410_probe", "worker", agent.RunInfo{}) + if got := acceptsRawInput(wrapped); got != raw { + t.Errorf("%T: AcceptsRawInput = %v, wrapped tool reports %v", wrapped, got, raw) + } + } + }) + + // The chain a worker's record_fact goes through on the server: the catalog + // decorator and the usage meter (ToolResolve, wireTools), then guardPanic + // (AugmentTools), then the harness itself. + t.Run("server chain", func(t *testing.T) { + dsn, _, err := db.DSN() + if err != nil { + db.SkipPGUnavailable(t, err) + } + pg, err := db.Open(dsn) + if err != nil { + db.SkipPGUnavailable(t, err) + } + defer pg.Close() + agent.SetToolAugment(nil) + wireTools(pg, nil) + task, err := pg.CreateTaskWithOptions("fix410", "record_fact through the wrapper chain", db.TaskCreateOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = pg.DeleteTask(task.ID) }) + ts := agent.NewToolSet(pg.Exploration(task.ExplorationID), "worker") + tools, _, cleanup := agent.AugmentTools(ctx, "worker", ts.WorkerTools()) + defer cleanup() + recordFact := names(tools)["record_fact"] + if recordFact == nil { + t.Fatal("worker has no record_fact") + } + if !acceptsRawInput(recordFact) { + t.Errorf("record_fact lost the harness opt-out behind %T", recordFact) + } + + // The first call is malformed (facts is not an array): the harness + // schema check answers it with its generic error unless the tool opted + // out. The second is the legacy single fact, which the published + // schema does not describe and the handler still accepts. + p := &toolScript{results: make(chan string, 4), calls: []scriptedCall{ + {"record_fact", `{"facts":{"summary":"not an array"}}`}, + {"record_fact", `{"summary":"legacy single fact"}`}, + }} + for _, err := range harness.Query(ctx, harness.QueryInput{ + Provider: p, + Tools: actool.NewRegistry(recordFact), + Messages: []llm.Message{llm.UserText("record the facts")}, + }, harness.QueryDeps{}) { + if err != nil { + t.Fatal(err) + } + } + if refusal := p.next(t); !strings.Contains(refusal, "invalid record_fact input: facts must be an array") { + t.Errorf("malformed call: want the handler's own refusal, not the SDK schema error, got: %s", refusal) + } + if legacy := p.next(t); strings.HasPrefix(legacy, "[error]") || ts.Writes().Facts != 1 { + t.Errorf("legacy single fact not written (facts=%d): %s", ts.Writes().Facts, legacy) + } + }) +} diff --git a/server/tool_usage.go b/server/tool_usage.go index 5f9e7c884..5e69cfb45 100644 --- a/server/tool_usage.go +++ b/server/tool_usage.go @@ -33,6 +33,14 @@ func meterTool(t actool.CoreTool, recorder toolUsageRecorder, toolKey, agentKey return &meteredTool{CoreTool: t, recorder: recorder, toolKey: toolKey, agentKey: agentKey, ri: ri} } +// AcceptsRawInput forwards the resolved tool's opt-out of the harness schema +// check; the embedded interface alone would hide it from the harness's type +// assertion. +func (m *meteredTool) AcceptsRawInput() bool { + r, ok := m.CoreTool.(interface{ AcceptsRawInput() bool }) + return ok && r.AcceptsRawInput() +} + func (m *meteredTool) Call(ctx context.Context, input json.RawMessage, tc *actool.ToolContext) (actool.Result, error) { if err := m.recorder.InsertToolUsage(&db.ToolUsage{ ToolKey: m.toolKey, diff --git a/server/tools_wire_test.go b/server/tools_wire_test.go index 89ebf2faf..7a168d8db 100644 --- a/server/tools_wire_test.go +++ b/server/tools_wire_test.go @@ -73,12 +73,19 @@ func TestWireTools(t *testing.T) { // Edit description + add a default to a scalar param, then confirm the resolved // worker tool reflects both. Restore from code default on cleanup. t.Cleanup(func() { _ = pg.UpsertToolForce(rf.Key, rf.Description, rf.Schema, mustJSON(rf.Agents)) }) + // Facts-only contract: the fact fields, and so the + // editable scalar annotations, live on the facts[] items. var schema map[string]any _ = json.Unmarshal(rf.Schema, &schema) - if props, ok := schema["properties"].(map[string]any); ok { - if conf, ok := props["confidence"].(map[string]any); ok { - conf["default"] = "inferred" - } + itemProps := func(sc map[string]any) map[string]any { + props, _ := sc["properties"].(map[string]any) + facts, _ := props["facts"].(map[string]any) + items, _ := facts["items"].(map[string]any) + ip, _ := items["properties"].(map[string]any) + return ip + } + if conf, ok := itemProps(schema)["confidence"].(map[string]any); ok { + conf["default"] = "inferred" } edited := mustJSON(schema) if err := pg.UpdateTool(rf.Key, "EDITED DESC", edited, mustJSON([]string{"worker"}), true); err != nil { @@ -96,9 +103,8 @@ func TestWireTools(t *testing.T) { // Default injection: omit confidence → handler input should carry the default. // We can't run the real handler (nil store), but InputSchema must show the default. sc := got.InputSchema() - props := sc["properties"].(map[string]any) - conf := props["confidence"].(map[string]any) - if conf["default"] != "inferred" { + conf, _ := itemProps(sc)["confidence"].(map[string]any) + if conf == nil || conf["default"] != "inferred" { t.Errorf("confidence.default = %v, want inferred", conf["default"]) } }