diff --git a/src/mastra/agents/security-research/skill-prompt.ts b/src/mastra/agents/security-research/skill-prompt.ts index e59feb73a..b0986dd1e 100644 --- a/src/mastra/agents/security-research/skill-prompt.ts +++ b/src/mastra/agents/security-research/skill-prompt.ts @@ -1,187 +1,80 @@ -export const SECURITY_RESEARCH_SKILL_IDS = [ - "risk-discovery", - "code-security-review", - "spec-implementation-review", - "bug-hunt-planner", - "web-inspection-forensics", - "threat-intake-analysis", - "proactive-security-strategy", - "reverse-engineering-analysis", - "exploitation-sandbox", - "network-reachability", -] as const; +import { + createProductSkillRegistry, + formatProductSkillDirectory, + type ProductSkillRegistry, +} from "../../../server/skills/product-skill-registry"; +import { + SECURITY_RESEARCH_SKILLS_ROOT, + securityResearchWorkspace, +} from "../../config/workspace"; -export type SecurityResearchSkillId = (typeof SECURITY_RESEARCH_SKILL_IDS)[number]; +export type SecurityResearchSkillId = string; -type SkillPromptDefinition = { - name: SecurityResearchSkillId; - useWhen: string; - apply: readonly string[]; - output: string; -}; +const workspaceSkills = securityResearchWorkspace.skills; +if (!workspaceSkills) { + throw new Error("Security research Workspace must configure product skills."); +} + +export const securityResearchPromptSkillRegistry = createProductSkillRegistry( + workspaceSkills, + SECURITY_RESEARCH_SKILLS_ROOT, +); -const skillPromptCatalog = { - "risk-discovery": { - name: "risk-discovery", - useWhen: - "Use for passive broad risk discovery over an authorized repo, package, release, local checkout, or supplied target evidence.", - apply: [ - "Scope the target, check memory first, build a system map, then rank structural, logical, critical, patch-drift, and CVSS-10-precursor risks.", - "For broad target scanning, sweep Discovery Lanes before deepening: input handling; access control and auth/session; information disclosure and debug; exposed files/directories; browser/API security posture.", - "Aim for 3-5 distinct Evidence-backed Discovery Lanes when the target supports them, and put unsupported lanes in Open Questions.", - "Produce durable Security Signals, Hypotheses, and memory-card-ready summaries without active exploitation.", - ], - output: - "Return Scope, Top Areas, Discovery Lane Coverage, Baseline Coverage, Patch-Drift Signals, Deprioritized Areas, and Open Questions.", - }, - "code-security-review": { - name: "code-security-review", - useWhen: - "Use for authorized repositories, local checkouts, source archives, and diffs when the task is passive code security review or PR-style changed-file review.", - apply: [ - "When source files, diffs, or artifact text are available, call security-code-review-scan first to build the deterministic candidate queue.", - "Start with deterministic candidate signals before expensive agent investigation: entry points, auth boundaries, raw SQL, SSRF-shaped fetches, file writes, command execution, dangerous HTML, secret handling, and agent tool definitions.", - "Keep project context compact: target shape, auth/session primitives, data stores, priority paths, and known false-positive patterns.", - "Treat candidates as review reasons, not findings; create findings only from evidence-backed analysis.", - "Use append-only scan, investigation, and revalidation records when summarizing coverage, and preserve no-finding coverage for clean changed files.", - "Do not run install, build, test, scripts, shell commands, network calls, or browser mutation without explicit approval.", - ], - output: - "Return Scope, Scan Summary, Candidate Queue, Findings, No-Finding Coverage, and exact Next Steps or approval needs.", - }, - "spec-implementation-review": { - name: "spec-implementation-review", - useWhen: - "Use when security relies on a protocol, specification, standard, formal precondition, API contract, parser grammar, sandbox boundary, or distributed-system guarantee.", - apply: [ - "Use primary-source citations with stable URL plus version, commit, or section. Treat the source as a requirement and the code as separate implementation evidence.", - "For every extracted invariant, classify it as enforced, partially_enforced, not-located, or not-applicable and trace it to files, symbols, guards, tests, callers, and fallbacks.", - "Review optimized, delegated, error, and fallback paths; generate composition tasks when a module assumes a property another module must establish.", - "Create a passive Code Review Candidate only when a cited precondition conflicts with concrete implementation evidence. Do not turn an unlocated guard into a finding by itself.", - "Keep target-specific advisory details and exploit recipes out of reusable instructions and organic-hunt inputs.", - ], - output: - "Return Scope, Invariant Matrix, Module Contract Cards, Composition Tasks, evidence-backed Candidates, No-Finding Coverage, and exact validation requirements.", - }, - "bug-hunt-planner": { - name: "bug-hunt-planner", - useWhen: - "Use when turning a target, scope, risk area, or hypothesis into a staged bug-hunting plan.", - apply: [ - "Use practitioner stage language: scoping, recon, discovery, enumeration, fingerprinting, mapping, scanning, probing, fuzzing, validation, impact, reporting, retesting, and monitoring.", - "For broad web/API hunts, follow the repeatable hunt spine: attack-surface mapping -> vulnerability-class triage -> business-logic/API review -> secrets and sensitive-data review -> reporting and proof.", - "Run the Observe -> Model -> Hypothesize -> Experiment -> Evaluate -> Act loop for each branch.", - "Convert each lead into an experiment with question, hypothesis, prediction, method or tool, safety limit, expected evidence, stop condition, and next branch.", - "For broad scans, sample across Discovery Lanes before drilling into one issue; include boring endpoints, header trust clues, duplicate parameters, object IDs, API schema clues, error messages, and exposed client/config artifacts as leads.", - "Mark every experiment as passive or approval-gated, and treat scanner output as lead generation rather than proof.", - ], - output: - "Return Scope, Stage, Hunt Spine Progress, Discovery Lane Coverage, ranked Leads, Experiments, Decision Tree, Monitoring, and exact approval needs.", - }, - "web-inspection-forensics": { - name: "web-inspection-forensics", - useWhen: - "Use for web or API targets, browser captures, HAR/PCAP/log artifacts, URL maps, crawling, endpoint discovery, JS bundle review, package hints, and light web forensics.", - apply: [ - "Define allowed hosts, auth state, rate limits, start URLs, capture IDs, artifact paths, and stop conditions.", - "Collect passive Digital Records first: headers, redirects, robots/sitemap/security.txt, TLS/DNS basics, HTML, linked assets, HARs, logs, and prior captures.", - "Build a URL Map with pages, forms, POST endpoints, APIs, JS bundles, source maps, package or version hints, technologies, protocols, parameters, object IDs, and third-party APIs.", - "Sweep the hunt spine: map attack surface, triage vulnerability classes, model business logic and APIs, inspect secrets and sensitive data, then assess proof and report readiness.", - "During broad sweeps, report Security Signals by Discovery Lane with surface, observed behavior, evidence, confidence, and next step.", - "Preserve debug endpoints, verbose errors, stack traces, framework banners, header anomalies, exposed files, client bundle clues, and config-looking artifacts as Evidence before interpreting them.", - "Prefer prior artifacts and static URL/API inventory before live probing; stop probing when results no longer add a new route, boundary, evidence path, or vulnerability class.", - ], - output: - "Return Scope, Digital Records, URL Map, Stats, Hunt Spine Progress, Discovery Lane Coverage, Security Signals, and ranked Hypotheses.", - }, - "threat-intake-analysis": { - name: "threat-intake-analysis", - useWhen: - "Use for suspicious SMS/email spam, phishing links, raw .eml headers, QR URLs, attachments, hashes, strange payloads, DNS/RDAP/WHOIS/redirect questions, C2 clues, and threat-analysis defense guidance.", - apply: [ - "Treat the submitted message, screenshot, .eml, local sample, hash, or provided URL as the intake target; do not assume links inside a message are authorized live targets.", - "Preserve originals and extract observables offline: URLs, domains, IPs, emails, phone numbers, headers, SPF/DKIM/DMARC results, attachment metadata, hashes, QR links, and encoded payload fragments.", - "Prefer passive enrichment: DNS/RDAP/WHOIS, certificate transparency, existing scan records, prior artifacts, and public intelligence only when privacy allows; do not submit private URLs or samples to public scanners without approval.", - "Treat live URL visits, redirect following, downloads, payload detonation, browser interaction, and C2 contact as approval-gated work requiring an isolated workspace or VM, scoped egress, capture requirements, and stop conditions.", - "Map delivery, redirect, payload, credential collection, exploitation, and C2 relationships with confidence labels, then produce concrete mailbox, identity, endpoint, network, and detection actions.", - ], - output: - "Return Scope, Intake Evidence, Indicators, Redirect/Delivery Chain, Payload/C2 Assessment, Defense Actions, and exact Approval Needs.", - }, - "proactive-security-strategy": { - name: "proactive-security-strategy", - useWhen: - "Use when the starting point is a CVE, advisory, commit, patch, version diff, changelog claim, public exploit writeup, crash report, or concrete vulnerability clue.", - apply: [ - "Anchor confirmed facts before inference: affected subjects and versions, vulnerability class, root cause, patched behavior, and evidence quality.", - "Extract the vulnerable primitive from patch or diff clues, such as parsing, authz, deserialization, request routing, sandbox escape, cache behavior, or file handling.", - "Offer passive exploration angles first: software class, usage patterns, official surface, unofficial public surface, patch diff, and exploit-chain mapping.", - "Treat local reproduction, active probing, and network-touching work as approval-gated.", - ], - output: "Return Security Signals, ranked Hypotheses, Subject relations, and Safety notes.", - }, - "reverse-engineering-analysis": { - name: "reverse-engineering-analysis", - useWhen: - "Use for binaries, firmware, mobile apps, WebAssembly, JavaScript bundles, .NET/JVM artifacts, decompilation, disassembly, instrumentation, and post-processing workflows.", - apply: [ - "Preserve artifact provenance with hashes, size, timestamps, source, permissions, container/package metadata, tool versions, and command stats.", - "Start static: file type, strings, symbols, imports/exports, sections, package manifests, entropy, source maps, bundle metadata, routes, and suspicious constants.", - "Choose an artifact-specific path: native binary, JS bundle, WebAssembly, .NET/JVM, firmware, or mobile, and post-process into smaller searchable artifacts.", - "Use isolated execution or emulation only after static triage and explicit approval where needed.", - ], - output: - "Return Artifact Inventory, Security Signals, ranked Hypotheses, Evidence references, and Open Gaps.", - }, - "exploitation-sandbox": { - name: "exploitation-sandbox", - useWhen: - "Use only after a hypothesis is well-founded and the user has explicitly authorized active validation for the target and technique.", - apply: [ - "Confirm authorization, scope boundaries, proof requirements, stop conditions, and Workspace Container or VM isolation before any active step.", - "Prefer minimum necessary impact, idempotent checks, local reproduction, read-only proof, and clear timestamps.", - "Capture every active request or command as Evidence with request/response or stdout, hashes, tool versions, timestamps, and stop condition reached.", - "Stop immediately on unexpected access, instability, out-of-scope data, missing approval, or unsupported isolation.", - ], - output: - "Return Authorization Summary, Hypothesis confirmed or refuted, Reproduction Steps, Evidence, Confidence, Cleanup, and Stop Condition.", - }, - "network-reachability": { - name: "network-reachability", - useWhen: - "Use when a network probe, DNS lookup, HTTP fetch, or nmap scan reports the target host or the internet is unreachable (DNS failure, connection refused, no route to host, host seems down, name resolution failure).", - apply: [ - "STOP further probing immediately; do not retry the same probe automatically.", - "Treat this as an environment or target-readiness blocker, not evidence that the target has a security weakness.", - "Report the exact failed operation, normalized error, target, timestamp, and the boundary from which reachability was attempted. Preserve raw diagnostics behind the in-chat details control.", - "Do not call ask_user merely to offer more scans against an unreachable host. End the run cleanly with one practical setup check, such as starting the target, correcting its address/network attachment, or retrying the same bounded baseline after readiness is restored.", - ], - output: - "Return a concise target-readiness blocker with the normalized error, diagnostic context, and one evidence-backed recovery step.", - }, -} satisfies Record; +/** + * Compact coordinator directory derived from the same reviewed Workspace + * snapshot used by model-facing lookup and execution-profile revisioning. + * Full skill bodies remain out of the prompt until a stage requests them. + */ +export async function formatSecurityResearchSkillDirectory( + registry: ProductSkillRegistry = securityResearchPromptSkillRegistry, +) { + const snapshot = await registry.list(); + return formatProductSkillDirectory(snapshot); +} -export function formatSecurityResearchSkillInstructions( - skillIds: readonly SecurityResearchSkillId[], +/** + * Load only the stage's selected procedures, bounded by the registry adapter. + * Visibility and required capabilities are revalidated at invocation time. + */ +export async function formatSecurityResearchSkillInstructions( + skillIds: readonly SecurityResearchSkillId[], + options: { + registry?: ProductSkillRegistry; + capabilities?: readonly string[]; + } = {}, ) { - return skillIds - .map((skillId) => { - const skill = skillPromptCatalog[skillId]; - return [ - `- ${skill.name}: ${skill.useWhen}`, - ...skill.apply.map((item) => ` - ${item}`), - ` - Output: ${skill.output}`, - ].join("\n"); - }) - .join("\n"); + const registry = options.registry ?? securityResearchPromptSkillRegistry; + const requested = [...new Set(skillIds)]; + const snapshot = await registry.list({ + includeDetails: true, + limit: 100, + capabilities: options.capabilities, + }); + const diagnostics = snapshot.diagnostics; + const skills = snapshot.skills.filter((skill) => + requested.includes(skill.id), + ); + const found = new Set(skills.map((skill) => skill.id)); + const missing = requested.filter((skillId) => !found.has(skillId)); + + if (diagnostics.length > 0 || missing.length > 0) { + return formatIncompleteSnapshot([ + ...diagnostics, + ...missing.map( + (skillId) => + `Requested product skill ${skillId} is unavailable or not applicable in the current Workspace snapshot.`, + ), + ]); + } + + return skills + .map((skill) => `## ${skill.id} [${skill.revision}]\n\n${skill.detail}`) + .join("\n\n"); } -/** Compact main-agent directory; full procedures are supplied on demand by SkillSearchProcessor. */ -export function formatSecurityResearchSkillDirectory(skillIds: readonly SecurityResearchSkillId[]) { - return skillIds - .map((skillId) => { - const skill = skillPromptCatalog[skillId]; - return `- ${skill.name}: ${skill.useWhen}`; - }) - .join("\n"); +function formatIncompleteSnapshot(diagnostics: readonly string[]) { + return [ + "Product skill discovery status: incomplete.", + ...[...new Set(diagnostics)].map((diagnostic) => `- ${diagnostic}`), + ].join("\n"); } diff --git a/src/mastra/agents/security-research/stage-agents.ts b/src/mastra/agents/security-research/stage-agents.ts index ff78386fe..b001c93ae 100644 --- a/src/mastra/agents/security-research/stage-agents.ts +++ b/src/mastra/agents/security-research/stage-agents.ts @@ -285,7 +285,14 @@ function stageModel(definition: StageDefinition) { }; } -function stageInstructions(definition: StageDefinition) { +async function stageInstructions( + definition: StageDefinition, + capabilities: readonly string[] = [], +) { + const skillInstructions = await formatSecurityResearchSkillInstructions( + definition.skillHints, + { capabilities }, + ); return `${commonStageInstructions(definition.stage)} Stage role: @@ -296,7 +303,7 @@ ${definition.responsibilities.map((item) => `- ${item}`).join("\n")} Relevant product skill instructions: Product skills describe reusable runtime research procedures from sandbox/skills. Apply the relevant details below when they fit the assigned task, but keep the output scoped to this stage and the user's authorization. -${formatSecurityResearchSkillInstructions(definition.skillHints)} +${skillInstructions} Output contract: ${definition.outputContract.map((item) => `- ${item}`).join("\n")} @@ -309,7 +316,11 @@ function createStageAgent(definition: StageDefinition) { id: definition.id, name: definition.name, description: definition.description, - instructions: stageInstructions(definition), + instructions: ({ requestContext }) => + stageInstructions( + definition, + readRuntimeSkillCapabilities(requestContext.get("runtimeSkillCapabilities")), + ), model: stageModel(definition), memory: createSecurityResearchStageMemory( definition.stage, @@ -334,6 +345,11 @@ function createStageAgent(definition: StageDefinition) { }); } +function readRuntimeSkillCapabilities(value: unknown) { + if (!Array.isArray(value)) return []; + return value.filter((item): item is string => typeof item === "string"); +} + export function buildSecurityResearchStageThreadId( projectId: string, stage: SecurityResearchStageId, diff --git a/src/mastra/config/editor.ts b/src/mastra/config/editor.ts index ead0ab822..c5623eea9 100644 --- a/src/mastra/config/editor.ts +++ b/src/mastra/config/editor.ts @@ -12,10 +12,6 @@ import { parse } from "llm-strings"; import { getDefaultModelFor } from "../../lib/models"; import { getSecurityResearchRenderPrompt } from "../../lib/render-json"; import { MARKDOWN_LITERAL_FORMATTING_INSTRUCTION } from "../../lib/security-chat/formatting-instructions"; -import { - formatSecurityResearchSkillDirectory, - SECURITY_RESEARCH_SKILL_IDS, -} from "../agents/security-research/skill-prompt"; export const SECURITY_RESEARCH_PROMPT_BLOCK_IDS = { operatingRules: "security-research-operating-rules", @@ -128,8 +124,7 @@ Optimize prompt quality first, but keep cost, speed, and safety/auditability vis For non-trivial work, turn the request into compact tasks with scope, expected evidence, approval boundary, stop condition, and useful output. Take the next safe approved action instead of waiting unnecessarily. Use an Observe -> Orient -> Decide -> Act loop after new evidence, failures, denials, or blocked work. -Search and load a full product procedure only when it is relevant. Available skills: -${formatSecurityResearchSkillDirectory(SECURITY_RESEARCH_SKILL_IDS)} +Use the product skill directory injected by the configured Mastra Workspace. Search and load a full product procedure only when it is relevant. Do not rely on a separate prompt-owned skill catalog. Delegate focused stage-sized work when useful, preserve evidence and uncertainty in the final summary, and do not claim a finding without evidence. Treat authorization, approval, target scope, and non-destructive operation as hard constraints. Make ordinary stage delegations self-contained and omit forked mode. Use a forked subagent only when explicitly necessary to continue from parent conversation context; it clones the parent thread and is not the normal delegation path. diff --git a/src/mastra/config/skill-search.ts b/src/mastra/config/skill-search.ts index 5d0795ed6..ef012c30b 100644 --- a/src/mastra/config/skill-search.ts +++ b/src/mastra/config/skill-search.ts @@ -1,50 +1,230 @@ import { - BaseProcessor, - type ProcessInputStepArgs, - type ProcessInputStepResult, - SkillSearchProcessor, + BaseProcessor, + type ProcessInputStepArgs, + type ProcessInputStepResult, } from "@mastra/core/processors"; +import { createTool } from "@mastra/core/tools"; +import { z } from "zod"; + import { readSecurityResearchRuntimeToolProfile } from "../../lib/security-chat/runtime-tool-profile"; -import { securityResearchWorkspace } from "./workspace"; +import { + createProductSkillRegistry, + formatProductSkillDirectory, + type ProductSkillRegistry, +} from "../../server/skills/product-skill-registry"; +import { + SECURITY_RESEARCH_SKILLS_ROOT, + securityResearchWorkspace, +} from "./workspace"; -const readRequestContext = (requestContext: ProcessInputStepArgs["requestContext"], key: string) => - requestContext?.get?.(key); +const readRequestContext = ( + requestContext: ProcessInputStepArgs["requestContext"], + key: string, +) => requestContext?.get?.(key); /** A foreground-command-only run deliberately exposes no overlapping discovery tools. */ export const shouldUseSecurityResearchSkillSearch = ( - requestContext: ProcessInputStepArgs["requestContext"], + requestContext: ProcessInputStepArgs["requestContext"], ) => - readSecurityResearchRuntimeToolProfile( - readRequestContext(requestContext, "runtimeToolProfile"), - ) !== "foreground-command-only"; - -class RuntimeToolProfileSkillSearchProcessor extends BaseProcessor<"security-research-skill-search"> { - readonly id = "security-research-skill-search" as const; - readonly name = "Security Research Skill Search"; - - constructor(private readonly delegate: SkillSearchProcessor) { - super(); - } - - async processInputStep(args: ProcessInputStepArgs): Promise { - if (!shouldUseSecurityResearchSkillSearch(args.requestContext)) { - return { tools: args.tools }; - } - return this.delegate.processInputStep(args); - } + readSecurityResearchRuntimeToolProfile( + readRequestContext(requestContext, "runtimeToolProfile"), + ) !== "foreground-command-only"; + +type LoadedSkill = { revision: string; instructions: string }; +type ThreadSkillState = { + snapshotKey: string; + loaded: Map; +}; + +/** + * On-demand skill tools backed by the reviewed product registry. Search and load + * both re-read request-scoped capabilities and revalidate through Workspace. + */ +export class RegistryBackedSkillSearchProcessor extends BaseProcessor<"security-research-skill-search"> { + readonly id = "security-research-skill-search" as const; + readonly name = "Security Research Skill Search"; + readonly providesSkillDiscovery = "on-demand" as const; + + private readonly threadState = new Map(); + + constructor(private readonly registry: ProductSkillRegistry) { + super(); + } + + async processInputStep( + args: ProcessInputStepArgs, + ): Promise { + if (!shouldUseSecurityResearchSkillSearch(args.requestContext)) { + return { tools: args.tools }; + } + + const capabilitiesForRequest = () => + readRuntimeSkillCapabilities( + readRequestContext(args.requestContext, "runtimeSkillCapabilities"), + ); + const capabilities = capabilitiesForRequest(); + const snapshot = await this.registry.list({ capabilities }); + const snapshotKey = [ + snapshot.status, + snapshot.revision ?? "incomplete", + capabilityKey(capabilities), + ].join(":"); + const threadId = readThreadId(args); + let state = this.threadState.get(threadId); + if (!state || state.snapshotKey !== snapshotKey) { + state = { snapshotKey, loaded: new Map() }; + this.threadState.set(threadId, state); + } + + args.messageList.addSystem( + `Product skill directory (Workspace-owned). Full bodies are available only through on-demand loading.\n${formatProductSkillDirectory(snapshot)}`, + ); + args.messageList.addSystem( + "To discover an applicable product procedure, call search_skills. Load only the procedure needed for the current task with load_skill.", + ); + for (const [skillName, skill] of state.loaded) { + args.messageList.addSystem( + `[Skill: ${skillName}; revision: ${skill.revision}]\n\n${skill.instructions}`, + ); + } + + const searchSkills = createTool({ + id: "search_skills", + description: + "Search reviewed product research skills applicable to the current runtime capabilities.", + inputSchema: z.object({ query: z.string() }), + outputSchema: z.object({ + results: z.array( + z.object({ + name: z.string(), + description: z.string(), + revision: z.string(), + }), + ), + status: z.enum(["complete", "incomplete"]), + message: z.string(), + }), + execute: async ({ query }) => { + const normalizedQuery = query.trim(); + if (!normalizedQuery) { + return { + results: [], + status: "complete" as const, + message: "A non-empty skill search query is required.", + }; + } + const current = await this.registry.list({ + query: normalizedQuery, + limit: 5, + capabilities: capabilitiesForRequest(), + }); + return { + results: current.skills.map((skill) => ({ + name: skill.id, + description: skill.description.slice(0, 150), + revision: skill.revision, + })), + status: current.status, + message: + current.status === "incomplete" + ? `Skill discovery is incomplete: ${current.diagnostics.join("; ")}` + : `Found ${current.skills.length} applicable skill(s).`, + }; + }, + }); + + const loadSkill = createTool({ + id: "load_skill", + description: + "Load one reviewed product skill after revalidating current visibility and capabilities.", + inputSchema: z.object({ skillName: z.string() }), + outputSchema: z.object({ + success: z.boolean(), + status: z.enum(["complete", "incomplete"]), + message: z.string(), + skillName: z.string().optional(), + revision: z.string().optional(), + }), + execute: async ({ skillName }) => { + const normalizedName = skillName.trim(); + const current = await this.registry.list({ + query: normalizedName, + includeDetails: true, + limit: 100, + capabilities: capabilitiesForRequest(), + }); + const skill = current.skills.find( + (entry) => entry.id === normalizedName, + ); + if (current.status !== "complete" || !skill) { + state.loaded.delete(normalizedName); + return { + success: false, + status: current.status, + message: + current.status === "incomplete" + ? `Skill discovery is incomplete: ${current.diagnostics.join("; ")}` + : `Skill "${normalizedName}" is unavailable or not applicable to the current runtime capabilities.`, + }; + } + + state.loaded.set(skill.id, { + revision: skill.revision, + instructions: skill.detail, + }); + return { + success: true, + status: "complete" as const, + message: `Skill "${skill.id}" loaded for the current snapshot.`, + skillName: skill.id, + revision: skill.revision, + }; + }, + }); + + return { + tools: { + ...(args.tools ?? {}), + search_skills: searchSkills, + load_skill: loadSkill, + }, + }; + } } -const nativeSecurityResearchSkillSearchProcessor = new SkillSearchProcessor({ - workspace: securityResearchWorkspace, - search: { - topK: 2, - minScore: 0, - }, - ttl: 60 * 60 * 1000, -}); - -export const securityResearchSkillSearchProcessor = new RuntimeToolProfileSkillSearchProcessor( - nativeSecurityResearchSkillSearchProcessor, +function readRuntimeSkillCapabilities(value: unknown) { + if (!Array.isArray(value)) return []; + return [ + ...new Set( + value.filter((item): item is string => typeof item === "string"), + ), + ].sort(); +} + +function capabilityKey(capabilities: readonly string[]) { + return capabilities.join(","); +} + +function readThreadId(args: ProcessInputStepArgs) { + for (const key of ["mastra__threadId", "threadId", "workspaceThreadId"]) { + const value = readRequestContext(args.requestContext, key); + if (typeof value === "string" && value.trim()) return value; + } + return "default"; +} + +const workspaceSkills = securityResearchWorkspace.skills; +if (!workspaceSkills) { + throw new Error("Security research Workspace must configure product skills."); +} +const productSkillRegistry = createProductSkillRegistry( + workspaceSkills, + SECURITY_RESEARCH_SKILLS_ROOT, ); -export const securityResearchSkillInputProcessors = [securityResearchSkillSearchProcessor]; +export const securityResearchSkillSearchProcessor = + new RegistryBackedSkillSearchProcessor(productSkillRegistry); + +export const securityResearchSkillInputProcessors = [ + securityResearchSkillSearchProcessor, +]; diff --git a/src/mastra/tools/product-skill-registry.ts b/src/mastra/tools/product-skill-registry.ts index 99044af59..5d6fabfcd 100644 --- a/src/mastra/tools/product-skill-registry.ts +++ b/src/mastra/tools/product-skill-registry.ts @@ -1,40 +1,72 @@ import { createTool } from "@mastra/core/tools"; import { z } from "zod"; -import { listProductSkills } from "../../server/skills/product-skill-registry"; +import { createProductSkillRegistry } from "../../server/skills/product-skill-registry"; +import { + SECURITY_RESEARCH_SKILLS_ROOT, + securityResearchWorkspace, +} from "../config/workspace"; const reviewStatusSchema = z.enum(["reviewed", "needs-review"]); +const workspaceSkills = securityResearchWorkspace.skills; +if (!workspaceSkills) { + throw new Error("Security research Workspace must configure product skills."); +} + +const productSkillRegistry = createProductSkillRegistry( + workspaceSkills, + SECURITY_RESEARCH_SKILLS_ROOT, +); + export const productSkillRegistryTool = createTool({ - id: "security-product-skill-registry", - description: - "Lists reviewed product-side security research skills from sandbox/skills with descriptions, search tags, review status, and optional excerpts. Passive local-only registry inspection.", - inputSchema: z.object({ - query: z.string().optional(), - includeDetails: z.boolean().optional(), - limit: z.number().optional(), - }), - outputSchema: z.object({ - root: z.string(), - skillCount: z.number(), - skills: z.array( - z.object({ - id: z.string(), - name: z.string(), - description: z.string(), - path: z.string(), - reviewStatus: reviewStatusSchema, - searchTags: z.array(z.string()), - detail: z.string(), - }), - ), - }), - execute: async (input) => - listProductSkills({ - ...(input.query ? { query: input.query } : {}), - ...(typeof input.includeDetails === "boolean" - ? { includeDetails: input.includeDetails } - : {}), - ...(typeof input.limit === "number" ? { limit: input.limit } : {}), - }), + id: "security-product-skill-registry", + description: + "Lists reviewed product-side security research skills from sandbox/skills with descriptions, search tags, review status, and optional excerpts. Passive local-only registry inspection.", + inputSchema: z.object({ + query: z.string().optional(), + includeDetails: z.boolean().optional(), + limit: z.number().optional(), + }), + outputSchema: z.object({ + root: z.string(), + status: z.enum(["complete", "incomplete"]), + revision: z.string().nullable(), + diagnostics: z.array(z.string()), + skillCount: z.number(), + skills: z.array( + z.object({ + id: z.string(), + name: z.string(), + description: z.string(), + path: z.string(), + revision: z.string(), + reviewStatus: reviewStatusSchema, + reviewedProvenance: z.literal("mastra-workspace"), + applicability: z.object({ + userInvocable: z.boolean(), + requiredCapabilities: z.array(z.string()), + }), + searchTags: z.array(z.string()), + detail: z.string(), + detailRevision: z.string().optional(), + }), + ), + }), + execute: async (input, context) => + productSkillRegistry.list({ + ...(input.query ? { query: input.query } : {}), + ...(typeof input.includeDetails === "boolean" + ? { includeDetails: input.includeDetails } + : {}), + ...(typeof input.limit === "number" ? { limit: input.limit } : {}), + capabilities: readRuntimeSkillCapabilities( + context?.requestContext?.get?.("runtimeSkillCapabilities"), + ), + }), }); + +function readRuntimeSkillCapabilities(value: unknown) { + if (!Array.isArray(value)) return []; + return value.filter((item): item is string => typeof item === "string"); +} diff --git a/src/server/research/skill-revision.ts b/src/server/research/skill-revision.ts index 1be8c496d..f1b922127 100644 --- a/src/server/research/skill-revision.ts +++ b/src/server/research/skill-revision.ts @@ -1,33 +1,31 @@ -import { createHash } from "node:crypto"; -import { readdir, readFile } from "node:fs/promises"; -import { relative, resolve } from "node:path"; +import { + SECURITY_RESEARCH_SKILLS_ROOT, + securityResearchWorkspace, +} from "../../mastra/config/workspace"; +import { + createProductSkillRegistry, + type ProductSkillRegistry, +} from "../skills/product-skill-registry"; -const skillsRoot = resolve(process.cwd(), "sandbox/skills"); -let cachedRevision: Promise | undefined; - -export function getSecurityResearchSkillRevision() { - return (cachedRevision ??= computeSkillRevision()); +const workspaceSkills = securityResearchWorkspace.skills; +if (!workspaceSkills) { + throw new Error("Security research Workspace must configure product skills."); } -async function computeSkillRevision() { - const files = await collectSkillFiles(skillsRoot); - const hash = createHash("sha256"); - for (const file of files.sort()) { - hash.update(relative(skillsRoot, file)); - hash.update("\0"); - hash.update(await readFile(file)); - hash.update("\0"); - } - return `sandbox-skills-sha256:${hash.digest("hex")}`; -} +const executionProfileSkillRegistry = createProductSkillRegistry( + workspaceSkills, + SECURITY_RESEARCH_SKILLS_ROOT, +); -async function collectSkillFiles(directory: string): Promise { - const entries = await readdir(directory, { withFileTypes: true }); - const files: string[] = []; - for (const entry of entries) { - const path = resolve(directory, entry.name); - if (entry.isDirectory()) files.push(...(await collectSkillFiles(path))); - else if (entry.isFile() && entry.name === "SKILL.md") files.push(path); - } - return files; +/** Resolve the current reviewed Workspace snapshot; never pin an empty or stale catalog. */ +export async function getSecurityResearchSkillRevision( + registry: ProductSkillRegistry = executionProfileSkillRegistry, +) { + const snapshot = await registry.list(); + if (snapshot.status !== "complete" || !snapshot.revision) { + throw new Error( + `Product skill discovery is incomplete: ${snapshot.diagnostics.join("; ") || "unknown discovery failure"}`, + ); + } + return snapshot.revision; } diff --git a/src/server/skills/product-skill-registry.ts b/src/server/skills/product-skill-registry.ts index 88569a781..314bf9c67 100644 --- a/src/server/skills/product-skill-registry.ts +++ b/src/server/skills/product-skill-registry.ts @@ -1,180 +1,389 @@ -import { readdir, readFile } from "node:fs/promises"; -import { join, relative, resolve, sep } from "node:path"; +import { createHash } from "node:crypto"; -export type ProductSkillReviewStatus = "reviewed" | "needs-review"; +import type { + Skill, + SkillMetadata, + WorkspaceSkills, +} from "@mastra/core/workspace"; + +export type ProductSkillReviewStatus = "reviewed"; +export type ProductSkillRegistryStatus = "complete" | "incomplete"; export type ProductSkillRegistryEntry = { - id: string; - name: string; - description: string; - path: string; - reviewStatus: ProductSkillReviewStatus; - searchTags: string[]; - detail: string; + id: string; + name: string; + description: string; + path: string; + revision: string; + reviewStatus: ProductSkillReviewStatus; + reviewedProvenance: "mastra-workspace"; + applicability: { + userInvocable: boolean; + requiredCapabilities: string[]; + }; + searchTags: string[]; + detail: string; + detailRevision?: string; }; export type ListProductSkillsInput = { - root?: string; - query?: string; - includeDetails?: boolean; - limit?: number; + query?: string; + includeDetails?: boolean; + limit?: number; + capabilities?: readonly string[]; }; export type ListProductSkillsResult = { - root: string; - skillCount: number; - skills: ProductSkillRegistryEntry[]; + root: string; + status: ProductSkillRegistryStatus; + revision: string | null; + diagnostics: string[]; + skillCount: number; + skills: ProductSkillRegistryEntry[]; +}; + +export type ProductSkillRegistry = { + list(input?: ListProductSkillsInput): Promise; }; -const DEFAULT_SKILLS_ROOT = resolve(process.cwd(), "sandbox", "skills"); -const registryCache = new Map>(); - -export async function listProductSkills( - input: ListProductSkillsInput = {}, -): Promise { - const root = input.root ? resolve(/* turbopackIgnore: true */ input.root) : DEFAULT_SKILLS_ROOT; - const limit = normalizeLimit(input.limit); - const query = normalizeQuery(input.query); - const entries = await readSkillIndex(root); - const skills = entries - .filter((skill) => !query || matchesQuery(skill, query)) - .sort((left, right) => left.id.localeCompare(right.id)) - .slice(0, limit); - - const outputSkills = input.includeDetails - ? await Promise.all(skills.map((skill) => readSkillDetail(root, skill))) - : skills; - - return { - root, - skillCount: query - ? entries.filter((skill) => matchesQuery(skill, query)).length - : entries.length, - skills: outputSkills, - }; -} - -function readSkillIndex(root: string) { - let cached = registryCache.get(root); - if (!cached) { - cached = readSkillIndexUncached(root); - registryCache.set(root, cached); - } - return cached; -} - -async function readSkillIndexUncached(root: string) { - const entries = await readdir(root, { withFileTypes: true }); - const skills: ProductSkillRegistryEntry[] = []; - for (const entry of entries) { - if (!entry.isDirectory()) { - continue; - } - const skillPath = join(root, entry.name, "SKILL.md"); - const skill = await readSkillFile(root, skillPath, entry.name); - if (skill) { - skills.push(skill); - } - } - return skills; -} - -async function readSkillFile(root: string, skillPath: string, id: string) { - try { - const markdown = await readFile(skillPath, "utf8"); - const frontmatter = parseFrontmatter(markdown); - const description = frontmatter.description || firstParagraph(markdown); - const name = frontmatter.name || id; - const reviewStatus: ProductSkillReviewStatus = - frontmatter.name && frontmatter.description ? "reviewed" : "needs-review"; - return { - id, - name, - description, - path: relative(root, skillPath).split(sep).join("/"), - reviewStatus, - searchTags: buildSearchTags(id, name, description), - detail: "", - }; - } catch { - return null; - } -} - -async function readSkillDetail( - root: string, - skill: ProductSkillRegistryEntry, -): Promise { - const markdown = await readFile(join(root, skill.path), "utf8").catch(() => ""); - return { - ...skill, - detail: markdown ? markdownWithoutFrontmatter(markdown).slice(0, 1600).trim() : "", - }; -} - -function parseFrontmatter(markdown: string) { - const match = /^---\r?\n([\s\S]*?)\r?\n---/.exec(markdown); - if (!match?.[1]) { - return {}; - } - return { - name: frontmatterValue(match[1], "name"), - description: frontmatterValue(match[1], "description"), - }; -} - -function frontmatterValue(frontmatter: string, key: string) { - const match = new RegExp(`^${key}:\\s*(.+)$`, "m").exec(frontmatter); - return match?.[1]?.trim().replace(/^["']|["']$/g, "") ?? ""; -} - -function markdownWithoutFrontmatter(markdown: string) { - return markdown.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n?/, ""); -} - -function firstParagraph(markdown: string) { - return ( - markdownWithoutFrontmatter(markdown) - .split(/\r?\n\r?\n/) - .map((paragraph) => paragraph.replace(/^#+\s*/, "").trim()) - .find(Boolean) ?? "" - ); +export function formatProductSkillDirectory(snapshot: ListProductSkillsResult) { + if (snapshot.status === "incomplete") { + return [ + "Product skill discovery status: incomplete.", + ...[...new Set(snapshot.diagnostics)].map( + (diagnostic) => `- ${diagnostic}`, + ), + ].join("\n"); + } + + return [ + `Product skill snapshot: ${snapshot.revision}`, + ...snapshot.skills.map( + (skill) => `- ${skill.id} [${skill.revision}]: ${skill.description}`, + ), + ].join("\n"); +} + +const DEFAULT_SKILLS_ROOT = "sandbox/skills"; +const MAX_SKILL_DETAIL_LENGTH = 1_600; + +/** + * Project-facing adapter over Mastra's WorkspaceSkills registry. + * + * This module deliberately owns no filesystem discovery or durable cache. Mastra + * remains responsible for discovery, refresh, validation, and full-body loading. + */ +export function createProductSkillRegistry( + workspaceSkills: WorkspaceSkills, + root = DEFAULT_SKILLS_ROOT, +): ProductSkillRegistry { + return { + async list(input: ListProductSkillsInput = {}) { + const capabilities = new Set(input.capabilities ?? []); + const diagnostics: string[] = []; + let metadata: SkillMetadata[]; + + try { + // Publication must invalidate Mastra's in-memory discovery state in every + // runtime mode. The reviewed product corpus is deliberately small, so a + // native refresh is preferable to an indefinitely stale process cache. + await refreshWorkspaceSkills(workspaceSkills); + metadata = await workspaceSkills.list(); + } catch (error) { + return incompleteResult(root, discoveryDiagnostic(error)); + } + + const runtimeSkills = metadata.filter(isRuntimeProductSkill); + + if (runtimeSkills.length === 0) { + return incompleteResult( + root, + "Mastra Workspace discovery returned no visible product research skills.", + ); + } + const visible = runtimeSkills.filter((skill) => + hasRequiredCapabilities(skill, capabilities), + ); + + const query = normalizeQuery(input.query); + const allEntries = await loadSnapshotEntries({ + workspaceSkills, + metadata: visible, + diagnostics, + }); + const revision = catalogRevision(allEntries); + const matchingEntries = allEntries.filter( + (skill) => !query || matchesQuery(skill, query), + ); + const selectedEntries = matchingEntries.slice( + 0, + normalizeLimit(input.limit), + ); + + const skills = input.includeDetails + ? await loadCurrentDetails({ + workspaceSkills, + entries: selectedEntries, + capabilities, + diagnostics, + }) + : selectedEntries; + + return { + root, + status: diagnostics.length > 0 ? "incomplete" : "complete", + revision, + diagnostics, + skillCount: matchingEntries.length, + skills, + }; + }, + }; +} + +function incompleteResult( + root: string, + diagnostic: string, +): ListProductSkillsResult { + return { + root, + status: "incomplete", + revision: null, + diagnostics: [diagnostic], + skillCount: 0, + skills: [], + }; +} + +async function loadCurrentDetails(input: { + workspaceSkills: WorkspaceSkills; + entries: ProductSkillRegistryEntry[]; + capabilities: ReadonlySet; + diagnostics: string[]; +}) { + const loaded: ProductSkillRegistryEntry[] = []; + let currentSkills: SkillMetadata[]; + try { + await refreshWorkspaceSkills(input.workspaceSkills); + currentSkills = await input.workspaceSkills.list(); + } catch (error) { + input.diagnostics.push( + `Skill bodies could not be revalidated against Mastra Workspace: ${errorMessage(error)}`, + ); + return loaded; + } + + for (const entry of input.entries) { + try { + // Revalidate both discovery and applicability immediately before loading. + const currentMetadata = currentSkills.find( + (candidate) => + candidate.name === entry.id && candidate.path === entry.path, + ); + if ( + !currentMetadata || + !isRuntimeProductSkill(currentMetadata) || + !hasRequiredCapabilities(currentMetadata, input.capabilities) + ) { + input.diagnostics.push( + `Skill ${entry.id} was no longer visible or applicable when its body was requested.`, + ); + continue; + } + + const skill = await input.workspaceSkills.get(currentMetadata.path); + if (!skill || !isRuntimeProductSkill(skill)) { + input.diagnostics.push( + `Skill ${entry.id} could not be loaded from Mastra Workspace.`, + ); + continue; + } + + const currentEntry = toRegistryEntry(skill); + if (currentEntry.revision !== entry.revision) { + input.diagnostics.push( + `Skill ${entry.id} changed after the registry snapshot; retry the lookup against the new revision.`, + ); + continue; + } + + loaded.push(withDetail(currentEntry, skill)); + } catch (error) { + input.diagnostics.push( + `Skill ${entry.id} could not be revalidated: ${errorMessage(error)}`, + ); + } + } + + return loaded; +} + +async function refreshWorkspaceSkills(workspaceSkills: WorkspaceSkills) { + // Mastra's current refresh() expects dynamic skill paths to have been resolved + // by initialization. maybeRefresh() performs that initialization first. + await workspaceSkills.maybeRefresh(); + await workspaceSkills.refresh(); +} + +async function loadSnapshotEntries(input: { + workspaceSkills: WorkspaceSkills; + metadata: SkillMetadata[]; + diagnostics: string[]; +}) { + const entries: ProductSkillRegistryEntry[] = []; + for (const metadata of input.metadata) { + try { + const skill = await input.workspaceSkills.get(metadata.path); + if (!skill || !isRuntimeProductSkill(skill)) { + input.diagnostics.push( + `Skill ${metadata.name} could not be resolved from the current Mastra Workspace snapshot.`, + ); + continue; + } + entries.push(toRegistryEntry(skill)); + } catch (error) { + input.diagnostics.push( + `Skill ${metadata.name} could not be fingerprinted: ${errorMessage(error)}`, + ); + } + } + return entries.sort((left, right) => left.id.localeCompare(right.id)); +} + +function toRegistryEntry(skill: SkillMetadata): ProductSkillRegistryEntry { + const requiredCapabilities = readRequiredCapabilities(skill); + const userInvocable = skill["user-invocable"] !== false; + const revision = hashJson({ + name: skill.name, + path: skill.path, + description: skill.description, + userInvocable, + requiredCapabilities, + metadata: skill.metadata ?? null, + instructions: "instructions" in skill ? skill.instructions : null, + }); + + return { + id: skill.name, + name: skill.name, + description: skill.description, + path: skill.path, + revision, + reviewStatus: "reviewed", + reviewedProvenance: "mastra-workspace", + applicability: { + userInvocable, + requiredCapabilities, + }, + searchTags: buildSearchTags( + skill.name, + skill.description, + ...requiredCapabilities, + ), + detail: "", + detailRevision: revision, + }; +} + +function withDetail( + entry: ProductSkillRegistryEntry, + skill: Skill, +): ProductSkillRegistryEntry { + const detail = skill.instructions.slice(0, MAX_SKILL_DETAIL_LENGTH).trim(); + return { + ...entry, + detail, + detailRevision: entry.revision, + }; +} + +function isRuntimeProductSkill(skill: SkillMetadata) { + const normalizedPath = skill.path.replaceAll("\\", "/"); + const pathSegments = normalizedPath.split("/"); + const visibility = readMetadataString(skill.metadata, "visibility"); + return !pathSegments.includes(".agents") && visibility !== "maintainer"; +} + +function hasRequiredCapabilities( + skill: SkillMetadata, + availableCapabilities: ReadonlySet, +) { + return readRequiredCapabilities(skill).every((capability) => + availableCapabilities.has(capability), + ); +} + +function readRequiredCapabilities(skill: SkillMetadata) { + const raw = skill.metadata?.requiredCapabilities; + if (!Array.isArray(raw)) return []; + return [ + ...new Set( + raw.filter((value): value is string => typeof value === "string"), + ), + ].sort(); +} + +function readMetadataString( + metadata: Record | undefined, + key: string, +) { + const value = metadata?.[key]; + return typeof value === "string" ? value : undefined; +} + +function discoveryDiagnostic(error: unknown) { + return `Mastra Workspace skill discovery is incomplete: ${errorMessage(error)}`; +} + +function errorMessage(error: unknown) { + return error instanceof Error ? error.message : String(error); +} + +function catalogRevision(skills: ProductSkillRegistryEntry[]) { + return hashJson( + skills.map((skill) => ({ id: skill.id, revision: skill.revision })), + ); +} + +function hashJson(value: unknown) { + return hashText(JSON.stringify(value)); +} + +function hashText(value: string) { + return createHash("sha256").update(value).digest("hex"); } function buildSearchTags(...values: string[]) { - const tags = new Set(); - for (const value of values) { - for (const token of value.toLowerCase().split(/[^a-z0-9]+/)) { - if (token.length >= 3) { - tags.add(token); - } - } - } - return [...tags].slice(0, 24); + const tags = new Set(); + for (const value of values) { + for (const token of value.toLowerCase().split(/[^a-z0-9]+/)) { + if (token.length >= 3) tags.add(token); + } + } + return [...tags].slice(0, 24); } function normalizeQuery(value: string | undefined) { - const query = value?.trim().toLowerCase(); - return query || undefined; + const query = value?.trim().toLowerCase(); + return query || undefined; } function matchesQuery(skill: ProductSkillRegistryEntry, query: string) { - const haystack = [ - skill.id, - skill.name, - skill.description, - skill.path, - skill.reviewStatus, - ...skill.searchTags, - ] - .join(" ") - .toLowerCase(); - return haystack.includes(query); + return [ + skill.id, + skill.name, + skill.description, + skill.path, + skill.reviewStatus, + ...skill.applicability.requiredCapabilities, + ...skill.searchTags, + ] + .join(" ") + .toLowerCase() + .includes(query); } function normalizeLimit(value: number | undefined) { - if (typeof value !== "number" || !Number.isFinite(value)) { - return 50; - } - return Math.max(1, Math.min(Math.floor(value), 100)); + if (typeof value !== "number" || !Number.isFinite(value)) return 50; + return Math.max(1, Math.min(Math.floor(value), 100)); } diff --git a/tests/integration/product-skill-registry.test.ts b/tests/integration/product-skill-registry.test.ts index 8efaa8936..dc4a34468 100644 --- a/tests/integration/product-skill-registry.test.ts +++ b/tests/integration/product-skill-registry.test.ts @@ -1,29 +1,371 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { RequestContext } from "@mastra/core/request-context"; +import { LocalFilesystem, Workspace } from "@mastra/core/workspace"; import { describe, expect, it } from "vitest"; +import { + formatSecurityResearchSkillDirectory, + formatSecurityResearchSkillInstructions, +} from "../../src/mastra/agents/security-research/skill-prompt"; +import { RegistryBackedSkillSearchProcessor } from "../../src/mastra/config/skill-search"; import { productSkillRegistryTool } from "../../src/mastra/tools/product-skill-registry"; -import type { ListProductSkillsResult } from "../../src/server/skills/product-skill-registry"; +import { getSecurityResearchSkillRevision } from "../../src/server/research/skill-revision"; +import { + createProductSkillRegistry, + type ListProductSkillsResult, +} from "../../src/server/skills/product-skill-registry"; const executeRegistryTool = async ( - input: Parameters>[0], + input: Parameters>[0], ) => { - if (!productSkillRegistryTool.execute) { - throw new Error("productSkillRegistryTool is missing execute."); - } - return productSkillRegistryTool.execute(input, {} as never) as Promise; + if (!productSkillRegistryTool.execute) { + throw new Error("productSkillRegistryTool is missing execute."); + } + return productSkillRegistryTool.execute( + input, + {} as never, + ) as Promise; }; describe("product skill registry", () => { - it("filters skills through the passive Mastra tool", async () => { - const result = await executeRegistryTool({ - query: "reverse", - includeDetails: true, - limit: 5, - }); - - expect(result.skillCount).toBeGreaterThan(0); - expect(result.skills.map((skill) => skill.id)).toEqual( - expect.arrayContaining(["reverse-engineering-analysis"]), - ); - expect(result.skills[0]?.detail).toMatch(/Reverse Engineering/i); - }); + it("filters skills through the passive Mastra tool", async () => { + const result = await executeRegistryTool({ + query: "reverse", + includeDetails: true, + limit: 5, + }); + + expect(result.status).toBe("complete"); + expect(result.revision).toMatch(/^[a-f0-9]{64}$/); + expect(result.skillCount).toBeGreaterThan(0); + expect(result.skills.map((skill) => skill.id)).toEqual( + expect.arrayContaining(["reverse-engineering-analysis"]), + ); + expect(result.skills[0]?.detail).toMatch(/Reverse Engineering/i); + expect(result.skills[0]?.reviewedProvenance).toBe("mastra-workspace"); + expect(result.skills[0]?.detailRevision).toMatch(/^[a-f0-9]{64}$/); + }); + + it("keeps Workspace lookup, prompt consumers, and execution profiles on one refreshed snapshot", async () => { + const fixtureRoot = await mkdtemp( + join(tmpdir(), "product-skill-registry-"), + ); + const skillsRoot = join(fixtureRoot, "sandbox", "skills"); + const maintainerRoot = join(fixtureRoot, ".agents", "skills"); + const skillDirectory = join(skillsRoot, "runtime-review"); + const maintainerDirectory = join(maintainerRoot, "maintainer-only"); + + await mkdir(skillsRoot, { recursive: true }); + await mkdir(maintainerDirectory, { recursive: true }); + await writeSkill(maintainerDirectory, { + name: "maintainer-only", + description: "Changes TypeScript in the application.", + body: "# Maintainer only\n\nDo not expose this procedure at runtime.", + }); + + const workspace = new Workspace({ + id: `product-skill-registry-${Date.now()}`, + filesystem: new LocalFilesystem({ basePath: fixtureRoot }), + skills: [skillsRoot, maintainerRoot], + checkSkillFileMtime: true, + }); + const workspaceSkills = workspace.skills; + if (!workspaceSkills) + throw new Error("Fixture Workspace is missing skills."); + const registry = createProductSkillRegistry(workspaceSkills, skillsRoot); + + try { + const incomplete = await registry.list(); + expect(incomplete).toMatchObject({ + status: "incomplete", + revision: null, + skillCount: 0, + skills: [], + }); + await expect( + formatSecurityResearchSkillDirectory(registry), + ).resolves.toContain("Product skill discovery status: incomplete."); + + await mkdir(skillDirectory, { recursive: true }); + await writeSkill(skillDirectory, { + name: "runtime-review", + description: "Review a runtime artifact.", + body: `# Runtime review\n\nFirst published procedure.\n\n${"bounded-context ".repeat(180)}END-OF-UNBOUNDED-BODY`, + }); + await workspaceSkills.addSkill?.(skillDirectory); + + const first = await registry.list({ includeDetails: true }); + expect(first.status).toBe("complete"); + expect(first.skills.map((skill) => skill.id)).toEqual(["runtime-review"]); + expect(first.skills[0]?.detail).toContain("First published procedure."); + expect(first.skills[0]?.detail.length).toBeLessThanOrEqual(1_600); + expect(first.skills[0]?.detail).not.toContain("END-OF-UNBOUNDED-BODY"); + expect(first.skills.some((skill) => skill.id === "maintainer-only")).toBe( + false, + ); + + const directory = await formatSecurityResearchSkillDirectory(registry); + const stageInstructions = await formatSecurityResearchSkillInstructions( + ["runtime-review"], + { registry }, + ); + const profileRevision = await getSecurityResearchSkillRevision(registry); + expect(directory).toContain(`Product skill snapshot: ${first.revision}`); + expect(directory).toContain( + `runtime-review [${first.skills[0]?.revision}]`, + ); + expect(stageInstructions).toContain( + `runtime-review [${first.skills[0]?.revision}]`, + ); + expect(stageInstructions).not.toContain("END-OF-UNBOUNDED-BODY"); + expect(profileRevision).toBe(first.revision); + + await writeSkill(skillDirectory, { + name: "runtime-review", + description: "Review a runtime artifact.", + body: "# Runtime review\n\nSecond published procedure.", + }); + + const second = await registry.list({ includeDetails: true }); + expect(second.status).toBe("complete"); + expect(second.revision).not.toBe(first.revision); + expect(second.skills[0]?.detailRevision).not.toBe( + first.skills[0]?.detailRevision, + ); + expect(second.skills[0]?.detail).toContain("Second published procedure."); + await expect( + formatSecurityResearchSkillDirectory(registry), + ).resolves.toContain(`Product skill snapshot: ${second.revision}`); + await expect(getSecurityResearchSkillRevision(registry)).resolves.toBe( + second.revision, + ); + } finally { + await rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("revalidates capability requirements before exposing a skill body", async () => { + const fixtureRoot = await mkdtemp( + join(tmpdir(), "product-skill-capability-"), + ); + const skillsRoot = join(fixtureRoot, "sandbox", "skills"); + const skillDirectory = join(skillsRoot, "device-review"); + await mkdir(skillDirectory, { recursive: true }); + await writeSkill(skillDirectory, { + name: "device-review", + description: "Review an authorized device image.", + body: "# Device review\n\nInspect the supplied image.", + requiredCapabilities: ["firmware-read"], + }); + + const workspace = new Workspace({ + id: `product-skill-capability-${Date.now()}`, + filesystem: new LocalFilesystem({ basePath: fixtureRoot }), + skills: [skillsRoot], + checkSkillFileMtime: true, + }); + const workspaceSkills = workspace.skills; + if (!workspaceSkills) + throw new Error("Fixture Workspace is missing skills."); + const registry = createProductSkillRegistry(workspaceSkills, skillsRoot); + + try { + await expect( + registry.list({ includeDetails: true, capabilities: [] }), + ).resolves.toMatchObject({ status: "complete", skills: [] }); + const allowed = await registry.list({ + includeDetails: true, + capabilities: ["firmware-read"], + }); + expect(allowed.skills.map((skill) => skill.id)).toEqual([ + "device-review", + ]); + + await writeSkill(skillDirectory, { + name: "device-review", + description: "Review an authorized device image.", + body: "# Device review\n\nInspect the supplied image.", + requiredCapabilities: ["hardware-lab"], + }); + + await expect( + registry.list({ + includeDetails: true, + capabilities: ["firmware-read"], + }), + ).resolves.toMatchObject({ status: "complete", skills: [] }); + await expect( + registry.list({ + includeDetails: true, + capabilities: ["hardware-lab"], + }), + ).resolves.toMatchObject({ + status: "complete", + skills: [ + expect.objectContaining({ + id: "device-review", + detail: expect.stringContaining("Inspect the supplied image."), + }), + ], + }); + } finally { + await rm(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("enforces capability changes at the model-facing search and load tools", async () => { + const fixtureRoot = await mkdtemp(join(tmpdir(), "product-skill-tools-")); + const skillsRoot = join(fixtureRoot, "sandbox", "skills"); + const skillDirectory = join(skillsRoot, "device-review"); + await mkdir(skillDirectory, { recursive: true }); + await writeSkill(skillDirectory, { + name: "device-review", + description: "Review an authorized device image.", + body: `# Device review\n\n${"bounded-device-procedure ".repeat(100)}DO-NOT-INJECT-TAIL`, + requiredCapabilities: ["firmware-read"], + }); + + const workspace = new Workspace({ + id: `product-skill-tools-${Date.now()}`, + filesystem: new LocalFilesystem({ basePath: fixtureRoot }), + skills: [skillsRoot], + checkSkillFileMtime: true, + }); + const workspaceSkills = workspace.skills; + if (!workspaceSkills) + throw new Error("Fixture Workspace is missing skills."); + const registry = createProductSkillRegistry(workspaceSkills, skillsRoot); + const processor = new RegistryBackedSkillSearchProcessor(registry); + const requestContext = new RequestContext([ + ["mastra__threadId", "capability-thread"], + ["runtimeSkillCapabilities", []], + ]); + + try { + const deniedMessages = new SystemMessageCollector(); + const denied = await processor.processInputStep( + processorArgs(requestContext, deniedMessages), + ); + await expect( + executeProcessorTool(denied, "search_skills", { query: "device" }), + ).resolves.toMatchObject({ results: [], status: "complete" }); + await expect( + executeProcessorTool(denied, "load_skill", { + skillName: "device-review", + }), + ).resolves.toMatchObject({ success: false, status: "complete" }); + + requestContext.set("runtimeSkillCapabilities", ["firmware-read"]); + const allowedMessages = new SystemMessageCollector(); + const allowed = await processor.processInputStep( + processorArgs(requestContext, allowedMessages), + ); + await expect( + executeProcessorTool(allowed, "search_skills", { query: "device" }), + ).resolves.toMatchObject({ + results: [expect.objectContaining({ name: "device-review" })], + status: "complete", + }); + await expect( + executeProcessorTool(allowed, "load_skill", { + skillName: "device-review", + }), + ).resolves.toMatchObject({ success: true, status: "complete" }); + + const loadedMessages = new SystemMessageCollector(); + await processor.processInputStep( + processorArgs(requestContext, loadedMessages), + ); + expect(loadedMessages.text()).toContain("[Skill: device-review;"); + expect(loadedMessages.text()).not.toContain("DO-NOT-INJECT-TAIL"); + + requestContext.set("runtimeSkillCapabilities", []); + const revokedMessages = new SystemMessageCollector(); + await processor.processInputStep( + processorArgs(requestContext, revokedMessages), + ); + expect(revokedMessages.text()).not.toContain("[Skill: device-review;"); + await expect( + executeProcessorTool(allowed, "load_skill", { + skillName: "device-review", + }), + ).resolves.toMatchObject({ success: false, status: "complete" }); + } finally { + await rm(fixtureRoot, { recursive: true, force: true }); + } + }); }); + +class SystemMessageCollector { + private readonly messages: string[] = []; + + addSystem(message: string | { content: unknown }) { + this.messages.push( + typeof message === "string" ? message : String(message.content), + ); + } + + text() { + return this.messages.join("\n"); + } +} + +function processorArgs( + requestContext: RequestContext, + messageList: SystemMessageCollector, +) { + return { + requestContext, + messageList, + messages: [], + systemMessages: [], + stepNumber: 0, + steps: [], + model: "test/model", + tools: {}, + state: {}, + retryCount: 0, + abort: () => { + throw new Error("Processor aborted unexpectedly."); + }, + } as never; +} + +async function executeProcessorTool( + result: { tools?: Record }, + toolName: string, + input: Record, +) { + const tool = result.tools?.[toolName] as + | { + execute?: (input: Record, context: unknown) => unknown; + } + | undefined; + if (!tool?.execute) + throw new Error(`Processor tool ${toolName} is unavailable.`); + return tool.execute(input, {}); +} + +async function writeSkill( + directory: string, + input: { + name: string; + description: string; + body: string; + requiredCapabilities?: string[]; + }, +) { + const metadata = input.requiredCapabilities + ? `metadata:\n requiredCapabilities:\n${input.requiredCapabilities + .map((capability) => ` - ${capability}`) + .join("\n")}\n` + : ""; + await writeFile( + join(directory, "SKILL.md"), + `---\nname: ${input.name}\ndescription: ${input.description}\n${metadata}---\n\n${input.body}\n`, + "utf8", + ); +} diff --git a/tests/integration/stage-agents.test.ts b/tests/integration/stage-agents.test.ts index 712d0f9b3..b88edf1c2 100644 --- a/tests/integration/stage-agents.test.ts +++ b/tests/integration/stage-agents.test.ts @@ -123,6 +123,15 @@ describe("security research stage agents", () => { }); }); + it("resolves stage procedures from the revisioned Workspace skill snapshot", async () => { + const instructions = await securityResearchStageAgents.securityReconAgent.getInstructions({ + requestContext: new RequestContext(), + }); + + expect(instructions).toEqual(expect.any(String)); + expect(instructions).toMatch(/## risk-discovery \[[a-f0-9]{64}\]/); + }); + it("uses the coordinator model for stage subagents when controller context omits modelUri", async () => { const requestContext = new RequestContext([ ["coordinatorModelUri", "llm://lmstudio/coordinator/stage-model?maxTokens=128000"],