Add LLM routing fallback evidence gates

skills/ai-security/llm-top-10/tests/benign/fail-closed-fallback-scoped-tools.ts

@@ -0,0 +1,40 @@
+type Route = {
+  model: string;
+  tools: string[];
+  maxOutputTokens: number;
+  temperature: number;
+  failMode: "fail_closed";
+  dataClass: "restricted" | "public";
+};
+
+const regulatedRoute: Route = {
+  model: "secure-reasoner-2026-05",
+  tools: ["retrieve_authorized_ticket", "draft_reply"],
+  maxOutputTokens: 700,
+  temperature: 0.1,
+  failMode: "fail_closed",
+  dataClass: "restricted",
+};
+
+const degradedRoute: Route = {
+  model: "secure-reasoner-2026-05-backup",
+  tools: ["retrieve_authorized_ticket", "draft_reply"],
+  maxOutputTokens: 700,
+  temperature: 0.1,
+  failMode: "fail_closed",
+  dataClass: "restricted",
+};
+
+declare function audit(event: string, payload: Record<string, string>): void;
+
+export function chooseRegulatedRoute(primaryHealthy: boolean): Route {
+  if (primaryHealthy) return regulatedRoute;
+  audit("llm_route_degraded", { from: regulatedRoute.model, to: degradedRoute.model });
+  return degradedRoute;
+}
+
+export function assertActionAllowed(route: Route, action: string) {
+  if (action === "send_reply" && !route.tools.includes(action)) {
+    throw new Error("Human approval required before side-effectful fallback action");
+  }
+}

skills/ai-security/llm-top-10/tests/benign/policy-preserving-context-fitting.yaml

@@ -0,0 +1,20 @@
+context_fitting:
+  non_truncatable:
+    - system_policy
+    - developer_policy
+    - tenant_id
+    - source_acl_labels
+    - data_classification
+    - denied_actions
+    - active_constraints
+  truncatable:
+    - untrusted_conversation_history
+    - retrieved_snippet_text
+  summarization:
+    label: untrusted_summary
+    preserve_metadata:
+      - source_acl_labels
+      - tenant_id
+      - data_classification
+  high_risk_actions:
+    on_missing_required_evidence: fail_closed

skills/ai-security/llm-top-10/tests/vulnerable/context-fitting-drops-auth.py

@@ -0,0 +1,17 @@
+def fit_context(messages, max_tokens):
+    while token_count(messages) > max_tokens:
+        messages.pop(0)
+    return messages
+
+
+messages = [
+    {"role": "system", "content": "Use tenant ACLs and never execute denied actions."},
+    {"role": "developer", "content": "Denied actions: send_wire, delete_customer"},
+    *conversation_history,
+    {"role": "user", "content": user_prompt},
+]
+
+client.responses.create(
+    model="llm-large-context",
+    input=fit_context(messages, 120000),
+)

skills/ai-security/llm-top-10/tests/vulnerable/fail-open-fallback-broad-tools.ts

@@ -0,0 +1,68 @@
+type Message = { role: "system" | "user" | "assistant"; content: string };
+type Ticket = { id: string; tenantId: string; body: string };
+type User = { id: string; tenantId: string };
+
+const primaryRoute = {
+  name: "regulated-support",
+  model: process.env.PRIMARY_LLM_MODEL ?? "secure-reasoner-2026-05",
+  tools: ["retrieve_authorized_ticket", "draft_reply"],
+  maxOutputTokens: 700,
+  temperature: 0.1,
+  dataClass: "restricted",
+};
+
+const fallbackRoute = {
+  name: "regulated-support-fallback",
+  model: process.env.FALLBACK_LLM_MODEL ?? "fast-latest",
+  tools: ["retrieve_authorized_ticket", "draft_reply", "send_reply"],
+  temperature: 0.9,
+  dataClass: "restricted",
+};
+
+declare const primary: {
+  responses: {
+    create(input: {
+      model: string;
+      input: Message[];
+      tools: string[];
+      max_output_tokens: number;
+      temperature: number;
+    }): Promise<unknown>;
+  };
+};
+
+declare const fallback: {
+  chat: {
+    completions: {
+      create(input: {
+        model: string;
+        messages: Message[];
+        tools: string[];
+        temperature: number;
+      }): Promise<unknown>;
+    };
+  };
+};
+
+declare function buildAuthorizedTicketMessages(ticket: Ticket, user: User): Message[];
+
+export async function answerTicket(ticket: Ticket, user: User) {
+  const input = buildAuthorizedTicketMessages(ticket, user);
+
+  try {
+    return await primary.responses.create({
+      model: primaryRoute.model,
+      input,
+      tools: primaryRoute.tools,
+      max_output_tokens: primaryRoute.maxOutputTokens,
+      temperature: primaryRoute.temperature,
+    });
+  } catch {
+    return await fallback.chat.completions.create({
+      model: fallbackRoute.model,
+      messages: input,
+      tools: fallbackRoute.tools,
+      temperature: fallbackRoute.temperature,
+    });
+  }
+}

skills/ai-security/llm-top-10/tests/vulnerable/mutable-model-alias.yaml

@@ -0,0 +1,12 @@
+llm_routes:
+  support:
+    model: provider/latest
+    fallback_model: provider/preview
+    runtime_model_id_export: false
+    alias_change_approval: false
+    security_regression_tests: []
+    tool_choice: auto
+    tools:
+      - retrieve_ticket
+      - draft_reply
+      - send_reply