From 6d322b59f5d688957f50e960f2b3f623c2c94fd8 Mon Sep 17 00:00:00 2001 From: Garrett Maring Date: Sun, 24 May 2026 20:04:11 -0300 Subject: [PATCH] V38 Gate 4: Prove Prompt Benchmark Coverage Add the source-safe V38PromptBenchmarkReport artifact for PromptPart and complete Prompt benchmarking across Reading, Conversation, and tool-definition surfaces. Wire the Gate 4 generator, checker, protocol exports, workflow checks, and protocol tests; update V38 specifications, roadmap, and documentation with the closure anchor. Validation: pnpm run check:v38-gate1; pnpm run check:v38-gate2; pnpm run check:v38-gate3; pnpm run check:v38-gate4; node --test --test-force-exit packages/protocol/test/v38-prompt-benchmark-report.test.js; pnpm --filter @bitcode/protocol typecheck; pnpm --filter @bitcode/protocol test; node scripts/run-bitcode-spec-quality.mjs --mode strict-version --version V38; pnpm --dir packages/prompts exec tsx src/benchmarking/cli.ts benchmark:report; git diff --cached --check. --- .bitcode/v38-prompt-benchmark-report.json | 1891 +++++++++++++++++ .github/workflows/bitcode-canon-quality.yml | 3 + .github/workflows/bitcode-gate-quality.yml | 6 + BITCODE_SPEC_V38.md | 5 +- BITCODE_SPEC_V38_DELTA.md | 9 +- BITCODE_SPEC_V38_NOTES.md | 11 +- BITCODE_SPEC_V38_PARITY_MATRIX.md | 27 +- README.md | 20 +- SPECIFICATIONS_ROADMAP.md | 5 +- package.json | 3 + packages/prompts/src/benchmarking/cli.ts | 50 +- packages/protocol/README.md | 11 + .../src/canonical/prompt-benchmark-report.js | 791 +++++++ packages/protocol/src/index.d.ts | 10 + packages/protocol/src/index.js | 12 + .../test/v38-prompt-benchmark-report.test.js | 110 + ...heck-v38-gate4-prompt-benchmark-report.mjs | 293 +++ .../generate-v38-prompt-benchmark-report.mjs | 60 + 18 files changed, 3289 insertions(+), 28 deletions(-) create mode 100644 .bitcode/v38-prompt-benchmark-report.json create mode 100644 packages/protocol/src/canonical/prompt-benchmark-report.js create mode 100644 packages/protocol/test/v38-prompt-benchmark-report.test.js create mode 100644 scripts/check-v38-gate4-prompt-benchmark-report.mjs create mode 100644 scripts/generate-v38-prompt-benchmark-report.mjs diff --git a/.bitcode/v38-prompt-benchmark-report.json b/.bitcode/v38-prompt-benchmark-report.json new file mode 100644 index 000000000..65964f726 --- /dev/null +++ b/.bitcode/v38-prompt-benchmark-report.json @@ -0,0 +1,1891 @@ +{ + "artifactId": "v38-prompt-benchmark-report", + "schemaId": "bitcode.v38.promptBenchmarkReport.v1", + "version": "V38", + "currentTarget": "V37", + "generatedAt": "2026-05-24T00:00:00.000Z", + "artifactPath": ".bitcode/v38-prompt-benchmark-report.json", + "sourceSafetyVerdict": "source-safe-prompt-benchmark-metadata", + "subjectKindIds": [ + "benchmark-infrastructure", + "PromptPart", + "Prompt", + "DocCodeToolPrompt" + ], + "metricIds": [ + "intent_alignment", + "semantic_clarity", + "token_efficiency", + "model_stability", + "task_success", + "response_quality", + "schema_conformance", + "source_boundary_preservation" + ], + "disclosureTierIds": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "benchmark_fixture_source_safe", + "expected_typed_output_quality_source_safe", + "benchmark_score_source_safe", + "raw_prompt_text_private", + "raw_provider_response_private" + ], + "forbiddenPayloadClasses": [ + "secret_values", + "provider_tokens", + "wallet_private_material", + "protected_source_payloads", + "raw_protected_prompts", + "raw_prompt_text", + "raw_model_responses_with_protected_source", + "unpaid_assetpack_source", + "settlement_private_payloads" + ], + "rows": [ + { + "rowId": "benchmark:runner-and-doc-comment-infrastructure", + "familyId": "benchmark_infrastructure", + "subjectKindId": "benchmark-infrastructure", + "label": "Prompt benchmark runner, suite types, and doc-comment metadata parser", + "surfaceIds": [ + "PromptPartBenchmarkSuite", + "PromptBenchmarkSuite", + "BenchmarkRunner" + ], + "sourceRoots": [ + "packages/prompts/src/benchmarking/runner.ts", + "packages/prompts/src/benchmarking/types.ts", + "packages/prompts/src/benchmarking/README.md", + "packages/prompts/src/developing/doc-comment-developing.ts" + ], + "requiredMetricIds": [ + "intent_alignment", + "semantic_clarity", + "token_efficiency", + "model_stability", + "task_success", + "response_quality", + "schema_conformance", + "source_boundary_preservation" + ], + "fixtureInputs": [ + { + "fixtureId": "fixture.promptpart.intent-alignment", + "label": "PromptPart intent-alignment scoring fixture", + "contextBindingIds": [ + "promptPartId", + "intent", + "benchmarkDefinitions" + ], + "expectedTypedOutputQualityIds": [ + "numeric_score_0_to_1", + "benchmark_model_identity", + "timestamp" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:67ed53ff3467b5329405d9e7", + "sourceSafe": true + }, + { + "fixtureId": "fixture.prompt.task-success", + "label": "Complete Prompt task-success scoring fixture", + "contextBindingIds": [ + "promptId", + "formattedPromptIdentity", + "fixtureScenario", + "expectedOutputSchemaId" + ], + "expectedTypedOutputQualityIds": [ + "numeric_score_0_to_1", + "task_success_reason", + "response_quality_reason" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:58d1a36792fc1b9658806f16", + "sourceSafe": true + } + ], + "expectedTypedOutputQualityIds": [ + "PromptPartBenchmarkResult", + "PromptBenchmarkResult", + "DevelopingBenchmarkResults" + ], + "requiredPredicateIds": [ + "benchmark.runner-defines-promptpart-benchmark", + "benchmark.runner-defines-complete-prompt-benchmark", + "benchmark.types-define-promptpart-suite", + "benchmark.types-define-complete-prompt-suite", + "benchmark.doc-comments-parse-benchmarks", + "benchmark.readme-documents-pbv" + ], + "disclosureTierIds": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "benchmark_fixture_source_safe", + "expected_typed_output_quality_source_safe", + "benchmark_score_source_safe", + "raw_prompt_text_private", + "raw_provider_response_private" + ], + "forbiddenPayloadClasses": [ + "secret_values", + "provider_tokens", + "wallet_private_material", + "protected_source_payloads", + "raw_protected_prompts", + "raw_prompt_text", + "raw_model_responses_with_protected_source", + "unpaid_assetpack_source", + "settlement_private_payloads" + ], + "protectedSourceVisible": false, + "rawPromptTextSerialized": false, + "rawProviderResponseSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "sourceSafetyClass": "source_safe_prompt_benchmark_metadata", + "rowRoot": "v38-prompt-benchmark-row:26c3d662702bcc6e06d91e18", + "sourceRootsPresent": true + }, + { + "rowId": "promptpart:generic-ptrr-failsafe-thricified-foundation", + "familyId": "promptpart_suite", + "subjectKindId": "PromptPart", + "label": "Generic PTRR, Failsafe, ThricifiedGeneration, formatting, and DocCode PromptParts", + "surfaceIds": [ + "PTRR agent generic PromptParts", + "FailsafeGenerationSequence generic PromptParts", + "ThricifiedGeneration generic PromptParts", + "DocCodeToolPrompt generic labels" + ], + "sourceRoots": [ + "packages/prompts/src/raw_promptparts/generic" + ], + "filePathIncludes": [ + "promptpart_generic_agent_", + "promptpart_generic_ptrr_", + "promptpart_generic_doccode_", + "promptpart_generic_formatting_" + ], + "requiredMetricIds": [ + "intent_alignment", + "semantic_clarity", + "token_efficiency", + "model_stability" + ], + "fixtureInputs": [ + { + "fixtureId": "fixture.generic-ptrr.reason-judge-structured-output", + "label": "Generic Reason/Judge/StructuredOutput PromptPart stability fixture", + "contextBindingIds": [ + "ptrrStepName", + "failsafeStageId", + "thricifiedGenerationStageId", + "schemaId" + ], + "expectedTypedOutputQualityIds": [ + "reasoning_clear", + "judgment_constrained", + "structured_output_json_only" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:56f775122e24bf32aac13f9e", + "sourceSafe": true + }, + { + "fixtureId": "fixture.generic-doccode.section-labels", + "label": "Generic DocCode section PromptPart fixture", + "contextBindingIds": [ + "toolName", + "toolPurpose", + "toolParameters", + "toolOutput" + ], + "expectedTypedOutputQualityIds": [ + "section_label_clear", + "tool_documentation_composable" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:abd404e2227bab8912506774", + "sourceSafe": true + } + ], + "expectedTypedOutputQualityIds": [ + "reason_prompt_clarity", + "judge_prompt_constraint", + "structured_output_schema_focus", + "tool_prompt_section_composability" + ], + "requiredPredicateIds": [ + "promptpart.generic-has-doc-comments", + "promptpart.generic-has-benchmark-definitions", + "promptpart.generic-exports-promptparts", + "promptpart.generic-semantic-division-present", + "promptpart.generic-current-version-present" + ], + "disclosureTierIds": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "benchmark_fixture_source_safe", + "expected_typed_output_quality_source_safe", + "benchmark_score_source_safe", + "raw_prompt_text_private", + "raw_provider_response_private" + ], + "forbiddenPayloadClasses": [ + "secret_values", + "provider_tokens", + "wallet_private_material", + "protected_source_payloads", + "raw_protected_prompts", + "raw_prompt_text", + "raw_model_responses_with_protected_source", + "unpaid_assetpack_source", + "settlement_private_payloads" + ], + "protectedSourceVisible": false, + "rawPromptTextSerialized": false, + "rawProviderResponseSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "sourceSafetyClass": "source_safe_prompt_benchmark_metadata", + "rowRoot": "v38-prompt-benchmark-row:24a0a239664d3ebbd1cf9b7f", + "sourceRootsPresent": true + }, + { + "rowId": "promptpart:read-need-comprehension-specific", + "familyId": "promptpart_suite", + "subjectKindId": "PromptPart", + "label": "ReadNeedComprehensionSynthesis specific PromptParts", + "surfaceIds": [ + "ReadNeedComprehensionSynthesis", + "ReadNeedComprehensionSynthesis.prompt.need-synthesis" + ], + "sourceRoots": [ + "packages/prompts/src/raw_promptparts/specific", + "packages/pipelines/asset-pack/src/read-need.ts" + ], + "filePathIncludes": [ + "promptpart_specific_agent_comprehendread", + "read-need.ts" + ], + "requiredMetricIds": [ + "intent_alignment", + "semantic_clarity", + "model_stability", + "schema_conformance", + "source_boundary_preservation" + ], + "fixtureInputs": [ + { + "fixtureId": "fixture.read-need.enterprise-request-to-need", + "label": "Enterprise Read Request to reviewable Need fixture", + "contextBindingIds": [ + "read.prompt", + "sourceConstraints", + "targetArtifactKinds", + "closureCriteria", + "feedbackHistory" + ], + "expectedTypedOutputQualityIds": [ + "need_exactly_matches_request", + "need_no_overreach", + "need_no_underreach", + "source_constraints_preserved" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:1ef8810fea10aecdf9e5b9bc", + "sourceSafe": true + }, + { + "fixtureId": "fixture.read-need.resynthesis-feedback", + "label": "Need resynthesis feedback fixture", + "contextBindingIds": [ + "previousNeed", + "readerFeedback", + "sourceRevision", + "measurementRoot" + ], + "expectedTypedOutputQualityIds": [ + "feedback_applied", + "measurement_inputs_preserved", + "reviewable_need_shape" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:c434bfdcbb7e7282b2f23dd5", + "sourceSafe": true + } + ], + "expectedTypedOutputQualityIds": [ + "ReadNeed", + "ReadNeedPricingMeasurementInputs", + "AcceptedReadNeed", + "ResynthesisRequestedReadNeed" + ], + "requiredPredicateIds": [ + "promptpart.read-need-has-doc-comments", + "promptpart.read-need-has-benchmark-definitions", + "promptpart.read-need-exports-promptparts", + "promptpart.read-need-contract-names-pipeline", + "promptpart.read-need-schema-names-output" + ], + "disclosureTierIds": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "benchmark_fixture_source_safe", + "expected_typed_output_quality_source_safe", + "benchmark_score_source_safe", + "raw_prompt_text_private", + "raw_provider_response_private" + ], + "forbiddenPayloadClasses": [ + "secret_values", + "provider_tokens", + "wallet_private_material", + "protected_source_payloads", + "raw_protected_prompts", + "raw_prompt_text", + "raw_model_responses_with_protected_source", + "unpaid_assetpack_source", + "settlement_private_payloads" + ], + "protectedSourceVisible": false, + "rawPromptTextSerialized": false, + "rawProviderResponseSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "sourceSafetyClass": "source_safe_prompt_benchmark_metadata", + "rowRoot": "v38-prompt-benchmark-row:ad9b5e04d006f7ec61ba84fd", + "sourceRootsPresent": true + }, + { + "rowId": "promptpart:read-fits-finding-specific", + "familyId": "promptpart_suite", + "subjectKindId": "PromptPart", + "label": "ReadFitsFindingSynthesis specific PromptParts for Finding Fits and AssetPack synthesis", + "surfaceIds": [ + "ReadFitsFindingSynthesis", + "ReadFitsFindingSynthesis.discovery.finding-fits", + "ReadFitsFindingSynthesis.implementation.asset-pack" + ], + "sourceRoots": [ + "packages/prompts/src/raw_promptparts/specific", + "packages/pipelines/asset-pack/src/depository-search.ts" + ], + "filePathIncludes": [ + "promptpart_specific_agent_assetpack", + "promptpart_specific_agent_planimplementation", + "promptpart_specific_agent_readiness", + "promptpart_specific_agent_createpullrequest", + "promptpart_specific_tool_websearch", + "depository-search.ts" + ], + "requiredMetricIds": [ + "intent_alignment", + "semantic_clarity", + "model_stability", + "schema_conformance", + "source_boundary_preservation" + ], + "fixtureInputs": [ + { + "fixtureId": "fixture.read-fits.find-many-candidates", + "label": "Find many above-threshold depository candidates fixture", + "contextBindingIds": [ + "acceptedReadNeed", + "searchChannels", + "embeddingPolicy", + "rankingThreshold" + ], + "expectedTypedOutputQualityIds": [ + "many_candidate_fit_ids", + "ranking_root_present", + "threshold_verdicts_present", + "selected_fit_provenance_present" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:126191d958fece025c06ab93", + "sourceSafe": true + }, + { + "fixtureId": "fixture.read-fits.assetpack-synthesis-preview", + "label": "Source-safe AssetPack synthesis preview fixture", + "contextBindingIds": [ + "fitDeposits", + "acceptedReadNeed", + "deliveryMechanismTemplate", + "sourceSafePreview" + ], + "expectedTypedOutputQualityIds": [ + "assetpack_source_hidden_before_settlement", + "preview_measurements_present", + "settlement_posture_separated" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:33829f65f93cdbaad9afb971", + "sourceSafe": true + } + ], + "expectedTypedOutputQualityIds": [ + "DepositoryFitsResult", + "AssetPackSynthesisOutput", + "ReadyToFinishOutput", + "AssetPackSourceSafePreview" + ], + "requiredPredicateIds": [ + "promptpart.read-fits-has-doc-comments", + "promptpart.read-fits-has-benchmark-definitions", + "promptpart.read-fits-exports-promptparts", + "promptpart.read-fits-contract-names-pipeline", + "promptpart.read-fits-depository-search-names-tools" + ], + "disclosureTierIds": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "benchmark_fixture_source_safe", + "expected_typed_output_quality_source_safe", + "benchmark_score_source_safe", + "raw_prompt_text_private", + "raw_provider_response_private" + ], + "forbiddenPayloadClasses": [ + "secret_values", + "provider_tokens", + "wallet_private_material", + "protected_source_payloads", + "raw_protected_prompts", + "raw_prompt_text", + "raw_model_responses_with_protected_source", + "unpaid_assetpack_source", + "settlement_private_payloads" + ], + "protectedSourceVisible": false, + "rawPromptTextSerialized": false, + "rawProviderResponseSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "sourceSafetyClass": "source_safe_prompt_benchmark_metadata", + "rowRoot": "v38-prompt-benchmark-row:820ad672dfe5adbb2aee4953", + "sourceRootsPresent": true + }, + { + "rowId": "prompt:reading-pipeline-agent-prompts", + "familyId": "complete_prompt_suite", + "subjectKindId": "Prompt", + "label": "Complete Reading pipeline agent Prompt registries", + "surfaceIds": [ + "ReadNeedComprehensionSynthesis.prompts", + "ReadFitsFindingSynthesis.prompts" + ], + "sourceRoots": [ + "packages/pipelines/asset-pack/src/agents/prompts", + "packages/pipelines/asset-pack/src/reading-pipeline-contract.ts" + ], + "requiredMetricIds": [ + "intent_alignment", + "semantic_clarity", + "task_success", + "response_quality", + "schema_conformance", + "source_boundary_preservation" + ], + "fixtureInputs": [ + { + "fixtureId": "fixture.reading.prompt-registry-composition", + "label": "Reading prompt registry composition fixture", + "contextBindingIds": [ + "agentPromptId", + "ptrrStepPromptId", + "promptPartNamespaces", + "interpolatedContextKeys" + ], + "expectedTypedOutputQualityIds": [ + "agent_prompt_composes", + "plan_try_refine_retry_prompts_present", + "generation_promptparts_present" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:9259dfb4acc0a6915b4a6396", + "sourceSafe": true + }, + { + "fixtureId": "fixture.reading.schema-bound-return-types", + "label": "Reading schema-bound output fixture", + "contextBindingIds": [ + "returnType", + "inputType", + "outputSchemaId", + "telemetryTargetId" + ], + "expectedTypedOutputQualityIds": [ + "typed_output_matches_schema", + "schema_verdict_available", + "repair_path_named" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:e2dbb69b7aca564be1a17308", + "sourceSafe": true + } + ], + "expectedTypedOutputQualityIds": [ + "ReadNeed", + "DepositoryFitsResult", + "AssetPackSynthesisOutput", + "ReadyToFinishOutput", + "AssetPackCompletionOutput" + ], + "requiredPredicateIds": [ + "prompt.reading-has-doc-comments", + "prompt.reading-has-benchmark-definitions", + "prompt.reading-constructs-prompts", + "prompt.reading-sets-promptparts", + "prompt.reading-requires-promptparts", + "prompt.reading-contract-has-template-ids" + ], + "disclosureTierIds": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "benchmark_fixture_source_safe", + "expected_typed_output_quality_source_safe", + "benchmark_score_source_safe", + "raw_prompt_text_private", + "raw_provider_response_private" + ], + "forbiddenPayloadClasses": [ + "secret_values", + "provider_tokens", + "wallet_private_material", + "protected_source_payloads", + "raw_protected_prompts", + "raw_prompt_text", + "raw_model_responses_with_protected_source", + "unpaid_assetpack_source", + "settlement_private_payloads" + ], + "protectedSourceVisible": false, + "rawPromptTextSerialized": false, + "rawProviderResponseSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "sourceSafetyClass": "source_safe_prompt_benchmark_metadata", + "rowRoot": "v38-prompt-benchmark-row:42411c8f180ffd8b1ecab3bf", + "sourceRootsPresent": true + }, + { + "rowId": "prompt:conversation-system-prompt", + "familyId": "complete_prompt_suite", + "subjectKindId": "Prompt", + "label": "Website Conversation system Prompt", + "surfaceIds": [ + "WebsiteConversations", + "BitcodeTerminalConversationSystemPrompt" + ], + "sourceRoots": [ + "packages/conversations-generics/src/prompts/BitcodeTerminalConversationSystemPrompt.ts" + ], + "requiredMetricIds": [ + "intent_alignment", + "semantic_clarity", + "task_success", + "response_quality", + "source_boundary_preservation" + ], + "fixtureInputs": [ + { + "fixtureId": "fixture.conversation.system-prompt-route-local", + "label": "Conversation route-local system behavior fixture", + "contextBindingIds": [ + "conversationId", + "sourceSelectorRefs", + "terminalHandoffContext" + ], + "expectedTypedOutputQualityIds": [ + "route_local_history_preserved", + "source_selector_boundary_named", + "handoff_summary_source_safe" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:b627473df50823bc00d026f6", + "sourceSafe": true + } + ], + "expectedTypedOutputQualityIds": [ + "conversation_response_source_safe", + "conversation_tool_call_posture", + "terminal_handoff_summary" + ], + "requiredPredicateIds": [ + "prompt.conversation-has-doc-comments", + "prompt.conversation-has-benchmark-definitions", + "prompt.conversation-extends-prompt", + "prompt.conversation-requires-hierarchy", + "prompt.conversation-sets-specific-promptparts" + ], + "disclosureTierIds": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "benchmark_fixture_source_safe", + "expected_typed_output_quality_source_safe", + "benchmark_score_source_safe", + "raw_prompt_text_private", + "raw_provider_response_private" + ], + "forbiddenPayloadClasses": [ + "secret_values", + "provider_tokens", + "wallet_private_material", + "protected_source_payloads", + "raw_protected_prompts", + "raw_prompt_text", + "raw_model_responses_with_protected_source", + "unpaid_assetpack_source", + "settlement_private_payloads" + ], + "protectedSourceVisible": false, + "rawPromptTextSerialized": false, + "rawProviderResponseSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "sourceSafetyClass": "source_safe_prompt_benchmark_metadata", + "rowRoot": "v38-prompt-benchmark-row:6db8a4c22bd4e71c396a5ce9", + "sourceRootsPresent": true + }, + { + "rowId": "prompt:tool-definition-doc-code-prompts", + "familyId": "complete_prompt_suite", + "subjectKindId": "DocCodeToolPrompt", + "label": "Doc-comment-backed tool-definition Prompts", + "surfaceIds": [ + "DocCodeToolPrompt", + "formatUsableTools", + "WebSearchDocCodeToolPrompt", + "BitcodeRepositoryEvidenceSearchDocCodeToolPrompt", + "CreatePullRequestDocCodeToolPrompt" + ], + "sourceRoots": [ + "packages/tools-generics/src/doc-code-tool/DocCodeToolPrompt.ts", + "packages/tools-generics/src/doc-code-tool/formatUsableTools.ts", + "packages/generic-tools/web-search/src/prompts", + "packages/generic-tools/vcs/src/prompts", + "packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.ts" + ], + "requiredMetricIds": [ + "intent_alignment", + "semantic_clarity", + "task_success", + "response_quality", + "source_boundary_preservation" + ], + "fixtureInputs": [ + { + "fixtureId": "fixture.tool-definition.final-prompt-injection", + "label": "DocCode tool documentation final prompt injection fixture", + "contextBindingIds": [ + "toolName", + "toolPurpose", + "toolCapabilities", + "toolParameters", + "toolOutput" + ], + "expectedTypedOutputQualityIds": [ + "tool_sections_present", + "doc_comment_prompt_attached", + "usable_tools_format_source_safe" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:1d58dcdf2bfcf2a4fb8c317d", + "sourceSafe": true + }, + { + "fixtureId": "fixture.tool-definition.delivery-tool-source-boundary", + "label": "Delivery and search tool source-boundary fixture", + "contextBindingIds": [ + "searchQuery", + "pullRequestDeliveryInput", + "disclosureTier" + ], + "expectedTypedOutputQualityIds": [ + "protected_source_not_serialized", + "tool_output_receipt_source_safe" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:6bbab1064f19d28d6f7c2425", + "sourceSafe": true + } + ], + "expectedTypedOutputQualityIds": [ + "ToolExecutionPrompt", + "ToolDefinitionPrompt", + "DocCodeToolPrompt", + "formatUsableToolsOutput", + "ToolPromptDefinitionSchemaVerdict" + ], + "requiredPredicateIds": [ + "prompt.tool-has-doc-comments", + "prompt.tool-has-benchmark-definitions", + "prompt.tool-extends-doccode-prompt", + "prompt.tool-format-usable-tools-present", + "prompt.tool-sets-promptparts", + "prompt.tool-source-boundary-fixtures-present" + ], + "disclosureTierIds": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "benchmark_fixture_source_safe", + "expected_typed_output_quality_source_safe", + "benchmark_score_source_safe", + "raw_prompt_text_private", + "raw_provider_response_private" + ], + "forbiddenPayloadClasses": [ + "secret_values", + "provider_tokens", + "wallet_private_material", + "protected_source_payloads", + "raw_protected_prompts", + "raw_prompt_text", + "raw_model_responses_with_protected_source", + "unpaid_assetpack_source", + "settlement_private_payloads" + ], + "protectedSourceVisible": false, + "rawPromptTextSerialized": false, + "rawProviderResponseSerialized": false, + "credentialsSerialized": false, + "unpaidAssetPackSourceVisible": false, + "sourceSafetyClass": "source_safe_prompt_benchmark_metadata", + "rowRoot": "v38-prompt-benchmark-row:61a3698c778d6d3c6a99c6c3", + "sourceRootsPresent": true + } + ], + "fixtureInputs": [ + { + "fixtureId": "fixture.promptpart.intent-alignment", + "label": "PromptPart intent-alignment scoring fixture", + "contextBindingIds": [ + "promptPartId", + "intent", + "benchmarkDefinitions" + ], + "expectedTypedOutputQualityIds": [ + "numeric_score_0_to_1", + "benchmark_model_identity", + "timestamp" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:67ed53ff3467b5329405d9e7", + "sourceSafe": true + }, + { + "fixtureId": "fixture.prompt.task-success", + "label": "Complete Prompt task-success scoring fixture", + "contextBindingIds": [ + "promptId", + "formattedPromptIdentity", + "fixtureScenario", + "expectedOutputSchemaId" + ], + "expectedTypedOutputQualityIds": [ + "numeric_score_0_to_1", + "task_success_reason", + "response_quality_reason" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:58d1a36792fc1b9658806f16", + "sourceSafe": true + }, + { + "fixtureId": "fixture.generic-ptrr.reason-judge-structured-output", + "label": "Generic Reason/Judge/StructuredOutput PromptPart stability fixture", + "contextBindingIds": [ + "ptrrStepName", + "failsafeStageId", + "thricifiedGenerationStageId", + "schemaId" + ], + "expectedTypedOutputQualityIds": [ + "reasoning_clear", + "judgment_constrained", + "structured_output_json_only" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:56f775122e24bf32aac13f9e", + "sourceSafe": true + }, + { + "fixtureId": "fixture.generic-doccode.section-labels", + "label": "Generic DocCode section PromptPart fixture", + "contextBindingIds": [ + "toolName", + "toolPurpose", + "toolParameters", + "toolOutput" + ], + "expectedTypedOutputQualityIds": [ + "section_label_clear", + "tool_documentation_composable" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:abd404e2227bab8912506774", + "sourceSafe": true + }, + { + "fixtureId": "fixture.read-need.enterprise-request-to-need", + "label": "Enterprise Read Request to reviewable Need fixture", + "contextBindingIds": [ + "read.prompt", + "sourceConstraints", + "targetArtifactKinds", + "closureCriteria", + "feedbackHistory" + ], + "expectedTypedOutputQualityIds": [ + "need_exactly_matches_request", + "need_no_overreach", + "need_no_underreach", + "source_constraints_preserved" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:1ef8810fea10aecdf9e5b9bc", + "sourceSafe": true + }, + { + "fixtureId": "fixture.read-need.resynthesis-feedback", + "label": "Need resynthesis feedback fixture", + "contextBindingIds": [ + "previousNeed", + "readerFeedback", + "sourceRevision", + "measurementRoot" + ], + "expectedTypedOutputQualityIds": [ + "feedback_applied", + "measurement_inputs_preserved", + "reviewable_need_shape" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:c434bfdcbb7e7282b2f23dd5", + "sourceSafe": true + }, + { + "fixtureId": "fixture.read-fits.find-many-candidates", + "label": "Find many above-threshold depository candidates fixture", + "contextBindingIds": [ + "acceptedReadNeed", + "searchChannels", + "embeddingPolicy", + "rankingThreshold" + ], + "expectedTypedOutputQualityIds": [ + "many_candidate_fit_ids", + "ranking_root_present", + "threshold_verdicts_present", + "selected_fit_provenance_present" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:126191d958fece025c06ab93", + "sourceSafe": true + }, + { + "fixtureId": "fixture.read-fits.assetpack-synthesis-preview", + "label": "Source-safe AssetPack synthesis preview fixture", + "contextBindingIds": [ + "fitDeposits", + "acceptedReadNeed", + "deliveryMechanismTemplate", + "sourceSafePreview" + ], + "expectedTypedOutputQualityIds": [ + "assetpack_source_hidden_before_settlement", + "preview_measurements_present", + "settlement_posture_separated" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:33829f65f93cdbaad9afb971", + "sourceSafe": true + }, + { + "fixtureId": "fixture.reading.prompt-registry-composition", + "label": "Reading prompt registry composition fixture", + "contextBindingIds": [ + "agentPromptId", + "ptrrStepPromptId", + "promptPartNamespaces", + "interpolatedContextKeys" + ], + "expectedTypedOutputQualityIds": [ + "agent_prompt_composes", + "plan_try_refine_retry_prompts_present", + "generation_promptparts_present" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:9259dfb4acc0a6915b4a6396", + "sourceSafe": true + }, + { + "fixtureId": "fixture.reading.schema-bound-return-types", + "label": "Reading schema-bound output fixture", + "contextBindingIds": [ + "returnType", + "inputType", + "outputSchemaId", + "telemetryTargetId" + ], + "expectedTypedOutputQualityIds": [ + "typed_output_matches_schema", + "schema_verdict_available", + "repair_path_named" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:e2dbb69b7aca564be1a17308", + "sourceSafe": true + }, + { + "fixtureId": "fixture.conversation.system-prompt-route-local", + "label": "Conversation route-local system behavior fixture", + "contextBindingIds": [ + "conversationId", + "sourceSelectorRefs", + "terminalHandoffContext" + ], + "expectedTypedOutputQualityIds": [ + "route_local_history_preserved", + "source_selector_boundary_named", + "handoff_summary_source_safe" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:b627473df50823bc00d026f6", + "sourceSafe": true + }, + { + "fixtureId": "fixture.tool-definition.final-prompt-injection", + "label": "DocCode tool documentation final prompt injection fixture", + "contextBindingIds": [ + "toolName", + "toolPurpose", + "toolCapabilities", + "toolParameters", + "toolOutput" + ], + "expectedTypedOutputQualityIds": [ + "tool_sections_present", + "doc_comment_prompt_attached", + "usable_tools_format_source_safe" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:1d58dcdf2bfcf2a4fb8c317d", + "sourceSafe": true + }, + { + "fixtureId": "fixture.tool-definition.delivery-tool-source-boundary", + "label": "Delivery and search tool source-boundary fixture", + "contextBindingIds": [ + "searchQuery", + "pullRequestDeliveryInput", + "disclosureTier" + ], + "expectedTypedOutputQualityIds": [ + "protected_source_not_serialized", + "tool_output_receipt_source_safe" + ], + "fixtureRoot": "v38-prompt-benchmark-fixture:6bbab1064f19d28d6f7c2425", + "sourceSafe": true + } + ], + "expectedTypedOutputQualityIds": [ + "AcceptedReadNeed", + "AssetPackCompletionOutput", + "AssetPackSourceSafePreview", + "AssetPackSynthesisOutput", + "DepositoryFitsResult", + "DevelopingBenchmarkResults", + "DocCodeToolPrompt", + "PromptBenchmarkResult", + "PromptPartBenchmarkResult", + "ReadNeed", + "ReadNeedPricingMeasurementInputs", + "ReadyToFinishOutput", + "ResynthesisRequestedReadNeed", + "ToolDefinitionPrompt", + "ToolExecutionPrompt", + "ToolPromptDefinitionSchemaVerdict", + "conversation_response_source_safe", + "conversation_tool_call_posture", + "formatUsableToolsOutput", + "judge_prompt_constraint", + "reason_prompt_clarity", + "structured_output_schema_focus", + "terminal_handoff_summary", + "tool_prompt_section_composability" + ], + "sourcePredicateResults": [ + { + "id": "benchmark:runner-and-doc-comment-infrastructure.source-roots-present", + "rowId": "benchmark:runner-and-doc-comment-infrastructure", + "sourcePath": "packages/prompts/src/benchmarking/runner.ts,packages/prompts/src/benchmarking/types.ts,packages/prompts/src/benchmarking/README.md,packages/prompts/src/developing/doc-comment-developing.ts", + "passed": true + }, + { + "id": "benchmark:runner-and-doc-comment-infrastructure.source-files-present", + "rowId": "benchmark:runner-and-doc-comment-infrastructure", + "sourcePath": "packages/prompts/src/benchmarking/runner.ts,packages/prompts/src/benchmarking/types.ts,packages/prompts/src/benchmarking/README.md,packages/prompts/src/developing/doc-comment-developing.ts", + "passed": true + }, + { + "id": "benchmark:runner-and-doc-comment-infrastructure.fixtures-source-safe", + "rowId": "benchmark:runner-and-doc-comment-infrastructure", + "sourcePath": "fixtureInputs", + "passed": true + }, + { + "id": "benchmark:runner-and-doc-comment-infrastructure.expected-typed-output-qualities-present", + "rowId": "benchmark:runner-and-doc-comment-infrastructure", + "sourcePath": "expectedTypedOutputQualityIds", + "passed": true + }, + { + "id": "benchmark:runner-and-doc-comment-infrastructure.disclosure-tiers-present", + "rowId": "benchmark:runner-and-doc-comment-infrastructure", + "sourcePath": "disclosureTierIds", + "passed": true + }, + { + "id": "benchmark.runner-defines-promptpart-benchmark", + "rowId": "benchmark:runner-and-doc-comment-infrastructure", + "sourcePath": "packages/prompts/src/benchmarking/runner.ts", + "passed": true + }, + { + "id": "benchmark.runner-defines-complete-prompt-benchmark", + "rowId": "benchmark:runner-and-doc-comment-infrastructure", + "sourcePath": "packages/prompts/src/benchmarking/runner.ts", + "passed": true + }, + { + "id": "benchmark.types-define-promptpart-suite", + "rowId": "benchmark:runner-and-doc-comment-infrastructure", + "sourcePath": "packages/prompts/src/benchmarking/types.ts", + "passed": true + }, + { + "id": "benchmark.types-define-complete-prompt-suite", + "rowId": "benchmark:runner-and-doc-comment-infrastructure", + "sourcePath": "packages/prompts/src/benchmarking/types.ts", + "passed": true + }, + { + "id": "benchmark.doc-comments-parse-benchmarks", + "rowId": "benchmark:runner-and-doc-comment-infrastructure", + "sourcePath": "packages/prompts/src/developing/doc-comment-developing.ts", + "passed": true + }, + { + "id": "benchmark.readme-documents-pbv", + "rowId": "benchmark:runner-and-doc-comment-infrastructure", + "sourcePath": "packages/prompts/src/benchmarking/README.md", + "passed": true + }, + { + "id": "promptpart:generic-ptrr-failsafe-thricified-foundation.source-roots-present", + "rowId": "promptpart:generic-ptrr-failsafe-thricified-foundation", + "sourcePath": "packages/prompts/src/raw_promptparts/generic", + "passed": true + }, + { + "id": "promptpart:generic-ptrr-failsafe-thricified-foundation.source-files-present", + "rowId": "promptpart:generic-ptrr-failsafe-thricified-foundation", + "sourcePath": "packages/prompts/src/raw_promptparts/generic", + "passed": true + }, + { + "id": "promptpart:generic-ptrr-failsafe-thricified-foundation.fixtures-source-safe", + "rowId": "promptpart:generic-ptrr-failsafe-thricified-foundation", + "sourcePath": "fixtureInputs", + "passed": true + }, + { + "id": "promptpart:generic-ptrr-failsafe-thricified-foundation.expected-typed-output-qualities-present", + "rowId": "promptpart:generic-ptrr-failsafe-thricified-foundation", + "sourcePath": "expectedTypedOutputQualityIds", + "passed": true + }, + { + "id": "promptpart:generic-ptrr-failsafe-thricified-foundation.disclosure-tiers-present", + "rowId": "promptpart:generic-ptrr-failsafe-thricified-foundation", + "sourcePath": "disclosureTierIds", + "passed": true + }, + { + "id": "promptpart.generic-has-doc-comments", + "rowId": "promptpart:generic-ptrr-failsafe-thricified-foundation", + "sourcePath": "packages/prompts/src/raw_promptparts/generic", + "passed": true + }, + { + "id": "promptpart.generic-has-benchmark-definitions", + "rowId": "promptpart:generic-ptrr-failsafe-thricified-foundation", + "sourcePath": "packages/prompts/src/raw_promptparts/generic", + "passed": true + }, + { + "id": "promptpart.generic-exports-promptparts", + "rowId": "promptpart:generic-ptrr-failsafe-thricified-foundation", + "sourcePath": "packages/prompts/src/raw_promptparts/generic", + "passed": true + }, + { + "id": "promptpart.generic-semantic-division-present", + "rowId": "promptpart:generic-ptrr-failsafe-thricified-foundation", + "sourcePath": "packages/prompts/src/raw_promptparts/generic", + "passed": true + }, + { + "id": "promptpart.generic-current-version-present", + "rowId": "promptpart:generic-ptrr-failsafe-thricified-foundation", + "sourcePath": "packages/prompts/src/raw_promptparts/generic", + "passed": true + }, + { + "id": "promptpart:read-need-comprehension-specific.source-roots-present", + "rowId": "promptpart:read-need-comprehension-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/read-need.ts", + "passed": true + }, + { + "id": "promptpart:read-need-comprehension-specific.source-files-present", + "rowId": "promptpart:read-need-comprehension-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/read-need.ts", + "passed": true + }, + { + "id": "promptpart:read-need-comprehension-specific.fixtures-source-safe", + "rowId": "promptpart:read-need-comprehension-specific", + "sourcePath": "fixtureInputs", + "passed": true + }, + { + "id": "promptpart:read-need-comprehension-specific.expected-typed-output-qualities-present", + "rowId": "promptpart:read-need-comprehension-specific", + "sourcePath": "expectedTypedOutputQualityIds", + "passed": true + }, + { + "id": "promptpart:read-need-comprehension-specific.disclosure-tiers-present", + "rowId": "promptpart:read-need-comprehension-specific", + "sourcePath": "disclosureTierIds", + "passed": true + }, + { + "id": "promptpart.read-need-has-doc-comments", + "rowId": "promptpart:read-need-comprehension-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/read-need.ts", + "passed": true + }, + { + "id": "promptpart.read-need-has-benchmark-definitions", + "rowId": "promptpart:read-need-comprehension-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/read-need.ts", + "passed": true + }, + { + "id": "promptpart.read-need-exports-promptparts", + "rowId": "promptpart:read-need-comprehension-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/read-need.ts", + "passed": true + }, + { + "id": "promptpart.read-need-semantic-division-present", + "rowId": "promptpart:read-need-comprehension-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/read-need.ts", + "passed": true + }, + { + "id": "promptpart.read-need-current-version-present", + "rowId": "promptpart:read-need-comprehension-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/read-need.ts", + "passed": true + }, + { + "id": "promptpart.read-need-contract-names-pipeline", + "rowId": "promptpart:read-need-comprehension-specific", + "sourcePath": "packages/pipelines/asset-pack/src/read-need.ts", + "passed": true + }, + { + "id": "promptpart.read-need-schema-names-output", + "rowId": "promptpart:read-need-comprehension-specific", + "sourcePath": "packages/pipelines/asset-pack/src/read-need.ts", + "passed": true + }, + { + "id": "promptpart:read-fits-finding-specific.source-roots-present", + "rowId": "promptpart:read-fits-finding-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/depository-search.ts", + "passed": true + }, + { + "id": "promptpart:read-fits-finding-specific.source-files-present", + "rowId": "promptpart:read-fits-finding-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/depository-search.ts", + "passed": true + }, + { + "id": "promptpart:read-fits-finding-specific.fixtures-source-safe", + "rowId": "promptpart:read-fits-finding-specific", + "sourcePath": "fixtureInputs", + "passed": true + }, + { + "id": "promptpart:read-fits-finding-specific.expected-typed-output-qualities-present", + "rowId": "promptpart:read-fits-finding-specific", + "sourcePath": "expectedTypedOutputQualityIds", + "passed": true + }, + { + "id": "promptpart:read-fits-finding-specific.disclosure-tiers-present", + "rowId": "promptpart:read-fits-finding-specific", + "sourcePath": "disclosureTierIds", + "passed": true + }, + { + "id": "promptpart.read-fits-has-doc-comments", + "rowId": "promptpart:read-fits-finding-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/depository-search.ts", + "passed": true + }, + { + "id": "promptpart.read-fits-has-benchmark-definitions", + "rowId": "promptpart:read-fits-finding-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/depository-search.ts", + "passed": true + }, + { + "id": "promptpart.read-fits-exports-promptparts", + "rowId": "promptpart:read-fits-finding-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/depository-search.ts", + "passed": true + }, + { + "id": "promptpart.read-fits-semantic-division-present", + "rowId": "promptpart:read-fits-finding-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/depository-search.ts", + "passed": true + }, + { + "id": "promptpart.read-fits-current-version-present", + "rowId": "promptpart:read-fits-finding-specific", + "sourcePath": "packages/prompts/src/raw_promptparts/specific,packages/pipelines/asset-pack/src/depository-search.ts", + "passed": true + }, + { + "id": "promptpart.read-fits-contract-names-pipeline", + "rowId": "promptpart:read-fits-finding-specific", + "sourcePath": "packages/pipelines/asset-pack/src/depository-search.ts", + "passed": true + }, + { + "id": "promptpart.read-fits-depository-search-names-tools", + "rowId": "promptpart:read-fits-finding-specific", + "sourcePath": "packages/pipelines/asset-pack/src/depository-search.ts", + "passed": true + }, + { + "id": "prompt:reading-pipeline-agent-prompts.source-roots-present", + "rowId": "prompt:reading-pipeline-agent-prompts", + "sourcePath": "packages/pipelines/asset-pack/src/agents/prompts,packages/pipelines/asset-pack/src/reading-pipeline-contract.ts", + "passed": true + }, + { + "id": "prompt:reading-pipeline-agent-prompts.source-files-present", + "rowId": "prompt:reading-pipeline-agent-prompts", + "sourcePath": "packages/pipelines/asset-pack/src/agents/prompts,packages/pipelines/asset-pack/src/reading-pipeline-contract.ts", + "passed": true + }, + { + "id": "prompt:reading-pipeline-agent-prompts.fixtures-source-safe", + "rowId": "prompt:reading-pipeline-agent-prompts", + "sourcePath": "fixtureInputs", + "passed": true + }, + { + "id": "prompt:reading-pipeline-agent-prompts.expected-typed-output-qualities-present", + "rowId": "prompt:reading-pipeline-agent-prompts", + "sourcePath": "expectedTypedOutputQualityIds", + "passed": true + }, + { + "id": "prompt:reading-pipeline-agent-prompts.disclosure-tiers-present", + "rowId": "prompt:reading-pipeline-agent-prompts", + "sourcePath": "disclosureTierIds", + "passed": true + }, + { + "id": "prompt.reading-has-doc-comments", + "rowId": "prompt:reading-pipeline-agent-prompts", + "sourcePath": "packages/pipelines/asset-pack/src/agents/prompts", + "passed": true + }, + { + "id": "prompt.reading-has-benchmark-definitions", + "rowId": "prompt:reading-pipeline-agent-prompts", + "sourcePath": "packages/pipelines/asset-pack/src/agents/prompts", + "passed": true + }, + { + "id": "prompt.reading-constructs-prompts", + "rowId": "prompt:reading-pipeline-agent-prompts", + "sourcePath": "packages/pipelines/asset-pack/src/agents/prompts", + "passed": true + }, + { + "id": "prompt.reading-sets-promptparts", + "rowId": "prompt:reading-pipeline-agent-prompts", + "sourcePath": "packages/pipelines/asset-pack/src/agents/prompts", + "passed": true + }, + { + "id": "prompt.reading-requires-promptparts", + "rowId": "prompt:reading-pipeline-agent-prompts", + "sourcePath": "packages/pipelines/asset-pack/src/agents/prompts", + "passed": true + }, + { + "id": "prompt.reading-contract-has-template-ids", + "rowId": "prompt:reading-pipeline-agent-prompts", + "sourcePath": "packages/pipelines/asset-pack/src/reading-pipeline-contract.ts", + "passed": true + }, + { + "id": "prompt:conversation-system-prompt.source-roots-present", + "rowId": "prompt:conversation-system-prompt", + "sourcePath": "packages/conversations-generics/src/prompts/BitcodeTerminalConversationSystemPrompt.ts", + "passed": true + }, + { + "id": "prompt:conversation-system-prompt.source-files-present", + "rowId": "prompt:conversation-system-prompt", + "sourcePath": "packages/conversations-generics/src/prompts/BitcodeTerminalConversationSystemPrompt.ts", + "passed": true + }, + { + "id": "prompt:conversation-system-prompt.fixtures-source-safe", + "rowId": "prompt:conversation-system-prompt", + "sourcePath": "fixtureInputs", + "passed": true + }, + { + "id": "prompt:conversation-system-prompt.expected-typed-output-qualities-present", + "rowId": "prompt:conversation-system-prompt", + "sourcePath": "expectedTypedOutputQualityIds", + "passed": true + }, + { + "id": "prompt:conversation-system-prompt.disclosure-tiers-present", + "rowId": "prompt:conversation-system-prompt", + "sourcePath": "disclosureTierIds", + "passed": true + }, + { + "id": "prompt.conversation-has-doc-comments", + "rowId": "prompt:conversation-system-prompt", + "sourcePath": "packages/conversations-generics/src/prompts/BitcodeTerminalConversationSystemPrompt.ts", + "passed": true + }, + { + "id": "prompt.conversation-has-benchmark-definitions", + "rowId": "prompt:conversation-system-prompt", + "sourcePath": "packages/conversations-generics/src/prompts/BitcodeTerminalConversationSystemPrompt.ts", + "passed": true + }, + { + "id": "prompt.conversation-extends-prompt", + "rowId": "prompt:conversation-system-prompt", + "sourcePath": "packages/conversations-generics/src/prompts/BitcodeTerminalConversationSystemPrompt.ts", + "passed": true + }, + { + "id": "prompt.conversation-requires-hierarchy", + "rowId": "prompt:conversation-system-prompt", + "sourcePath": "packages/conversations-generics/src/prompts/BitcodeTerminalConversationSystemPrompt.ts", + "passed": true + }, + { + "id": "prompt.conversation-sets-specific-promptparts", + "rowId": "prompt:conversation-system-prompt", + "sourcePath": "packages/conversations-generics/src/prompts/BitcodeTerminalConversationSystemPrompt.ts", + "passed": true + }, + { + "id": "prompt:tool-definition-doc-code-prompts.source-roots-present", + "rowId": "prompt:tool-definition-doc-code-prompts", + "sourcePath": "packages/tools-generics/src/doc-code-tool/DocCodeToolPrompt.ts,packages/tools-generics/src/doc-code-tool/formatUsableTools.ts,packages/generic-tools/web-search/src/prompts,packages/generic-tools/vcs/src/prompts,packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.ts", + "passed": true + }, + { + "id": "prompt:tool-definition-doc-code-prompts.source-files-present", + "rowId": "prompt:tool-definition-doc-code-prompts", + "sourcePath": "packages/tools-generics/src/doc-code-tool/DocCodeToolPrompt.ts,packages/tools-generics/src/doc-code-tool/formatUsableTools.ts,packages/generic-tools/web-search/src/prompts,packages/generic-tools/vcs/src/prompts,packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.ts", + "passed": true + }, + { + "id": "prompt:tool-definition-doc-code-prompts.fixtures-source-safe", + "rowId": "prompt:tool-definition-doc-code-prompts", + "sourcePath": "fixtureInputs", + "passed": true + }, + { + "id": "prompt:tool-definition-doc-code-prompts.expected-typed-output-qualities-present", + "rowId": "prompt:tool-definition-doc-code-prompts", + "sourcePath": "expectedTypedOutputQualityIds", + "passed": true + }, + { + "id": "prompt:tool-definition-doc-code-prompts.disclosure-tiers-present", + "rowId": "prompt:tool-definition-doc-code-prompts", + "sourcePath": "disclosureTierIds", + "passed": true + }, + { + "id": "prompt.tool-has-doc-comments", + "rowId": "prompt:tool-definition-doc-code-prompts", + "sourcePath": "packages/tools-generics/src/doc-code-tool/DocCodeToolPrompt.ts,packages/tools-generics/src/doc-code-tool/formatUsableTools.ts,packages/generic-tools/web-search/src/prompts,packages/generic-tools/vcs/src/prompts,packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.ts", + "passed": true + }, + { + "id": "prompt.tool-has-benchmark-definitions", + "rowId": "prompt:tool-definition-doc-code-prompts", + "sourcePath": "packages/tools-generics/src/doc-code-tool/DocCodeToolPrompt.ts,packages/tools-generics/src/doc-code-tool/formatUsableTools.ts,packages/generic-tools/web-search/src/prompts,packages/generic-tools/vcs/src/prompts,packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.ts", + "passed": true + }, + { + "id": "prompt.tool-extends-doccode-prompt", + "rowId": "prompt:tool-definition-doc-code-prompts", + "sourcePath": "packages/tools-generics/src/doc-code-tool/DocCodeToolPrompt.ts,packages/tools-generics/src/doc-code-tool/formatUsableTools.ts,packages/generic-tools/web-search/src/prompts,packages/generic-tools/vcs/src/prompts,packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.ts", + "passed": true + }, + { + "id": "prompt.tool-format-usable-tools-present", + "rowId": "prompt:tool-definition-doc-code-prompts", + "sourcePath": "packages/tools-generics/src/doc-code-tool/formatUsableTools.ts", + "passed": true + }, + { + "id": "prompt.tool-sets-promptparts", + "rowId": "prompt:tool-definition-doc-code-prompts", + "sourcePath": "packages/tools-generics/src/doc-code-tool/DocCodeToolPrompt.ts,packages/tools-generics/src/doc-code-tool/formatUsableTools.ts,packages/generic-tools/web-search/src/prompts,packages/generic-tools/vcs/src/prompts,packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.ts", + "passed": true + }, + { + "id": "prompt.tool-source-boundary-fixtures-present", + "rowId": "prompt:tool-definition-doc-code-prompts", + "sourcePath": "fixtureInputs", + "passed": true + } + ], + "sourceEvidence": [ + { + "sourcePath": "packages/conversations-generics/src/prompts/BitcodeTerminalConversationSystemPrompt.ts", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.ts", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/generic-tools/vcs/src/prompts", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/generic-tools/web-search/src/prompts", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/pipelines/asset-pack/src/agents/prompts", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/pipelines/asset-pack/src/depository-search.ts", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/pipelines/asset-pack/src/read-need.ts", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/pipelines/asset-pack/src/reading-pipeline-contract.ts", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/prompts/src/benchmarking/README.md", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/prompts/src/benchmarking/runner.ts", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/prompts/src/benchmarking/types.ts", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/prompts/src/developing/doc-comment-developing.ts", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/prompts/src/raw_promptparts/generic", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/prompts/src/raw_promptparts/specific", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/tools-generics/src/doc-code-tool/DocCodeToolPrompt.ts", + "exists": true, + "legacy": false + }, + { + "sourcePath": "packages/tools-generics/src/doc-code-tool/formatUsableTools.ts", + "exists": true, + "legacy": false + } + ], + "sourceStatsByRow": { + "benchmark:runner-and-doc-comment-infrastructure": { + "rowId": "benchmark:runner-and-doc-comment-infrastructure", + "sourceRootExists": true, + "sourceFileCount": 4, + "sourceFileSample": [ + "packages/prompts/src/benchmarking/README.md", + "packages/prompts/src/benchmarking/runner.ts", + "packages/prompts/src/benchmarking/types.ts", + "packages/prompts/src/developing/doc-comment-developing.ts" + ], + "promptPartDocCommentCount": 4, + "promptDocCommentCount": 4, + "benchmarkDefinitionCount": 2, + "intentMetadataCount": 2, + "currentVersionMetadataCount": 0, + "versionsMetadataCount": 2, + "promptPartExportCount": 1, + "promptConstructionCount": 0, + "promptSetCallCount": 0, + "promptRequireCallCount": 0, + "promptRequiresHierarchyCount": 0, + "extendsPromptCount": 2, + "extendsDocCodeToolPromptCount": 0, + "docCodeToolPromptCount": 0, + "formatUsableToolsCount": 0, + "readNeedComprehensionNamesCount": 0, + "readFitsFindingNamesCount": 0, + "depositorySearchToolNamesCount": 0, + "promptTemplateIdCount": 0, + "benchmarkRunnerPromptPartMethodsCount": 1, + "benchmarkRunnerPromptMethodsCount": 1, + "promptPartSuiteTypeCount": 1, + "promptSuiteTypeCount": 1, + "extractBenchmarksCount": 3, + "pbvDocumentationCount": 31 + }, + "promptpart:generic-ptrr-failsafe-thricified-foundation": { + "rowId": "promptpart:generic-ptrr-failsafe-thricified-foundation", + "sourceRootExists": true, + "sourceFileCount": 59, + "sourceFileSample": [ + "packages/prompts/src/raw_promptparts/generic/promptpart_generic_agent_failsafe_chunk.ts", + "packages/prompts/src/raw_promptparts/generic/promptpart_generic_agent_failsafe_prepare_context.ts", + "packages/prompts/src/raw_promptparts/generic/promptpart_generic_agent_failsafe_stitch.ts", + "packages/prompts/src/raw_promptparts/generic/promptpart_generic_agent_failsafe_sum.ts", + "packages/prompts/src/raw_promptparts/generic/promptpart_generic_agent_generation_if_unknown_empty.ts", + "packages/prompts/src/raw_promptparts/generic/promptpart_generic_agent_generation_json_only_header.ts", + "packages/prompts/src/raw_promptparts/generic/promptpart_generic_agent_generation_json_only_single_object.ts", + "packages/prompts/src/raw_promptparts/generic/promptpart_generic_agent_generation_judge.ts", + "packages/prompts/src/raw_promptparts/generic/promptpart_generic_agent_generation_reason.ts", + "packages/prompts/src/raw_promptparts/generic/promptpart_generic_agent_generation_structured_output.ts", + "packages/prompts/src/raw_promptparts/generic/promptpart_generic_agent_generation_top_level_keys_hint.ts", + "packages/prompts/src/raw_promptparts/generic/promptpart_generic_agent_generation_use_this_structured_schema.ts" + ], + "promptPartDocCommentCount": 87, + "promptDocCommentCount": 0, + "benchmarkDefinitionCount": 87, + "intentMetadataCount": 87, + "currentVersionMetadataCount": 87, + "versionsMetadataCount": 87, + "promptPartExportCount": 59, + "promptConstructionCount": 0, + "promptSetCallCount": 0, + "promptRequireCallCount": 0, + "promptRequiresHierarchyCount": 0, + "extendsPromptCount": 0, + "extendsDocCodeToolPromptCount": 0, + "docCodeToolPromptCount": 0, + "formatUsableToolsCount": 0, + "readNeedComprehensionNamesCount": 0, + "readFitsFindingNamesCount": 0, + "depositorySearchToolNamesCount": 0, + "promptTemplateIdCount": 0, + "benchmarkRunnerPromptPartMethodsCount": 0, + "benchmarkRunnerPromptMethodsCount": 0, + "promptPartSuiteTypeCount": 0, + "promptSuiteTypeCount": 0, + "extractBenchmarksCount": 0, + "pbvDocumentationCount": 0 + }, + "promptpart:read-need-comprehension-specific": { + "rowId": "promptpart:read-need-comprehension-specific", + "sourceRootExists": true, + "sourceFileCount": 16, + "sourceFileSample": [ + "packages/pipelines/asset-pack/src/read-need.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_comprehendread_context_awareness_detailcontent.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_comprehendread_phase_plan_label.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_comprehendread_phase_refine_label.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_comprehendread_phase_retry_label.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_comprehendread_phase_try_label.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_comprehendread_plan_analysis.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_comprehendread_plan_strategy.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_comprehendread_refine_assessment.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_comprehendread_refine_optimization.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_comprehendread_retry_errorhandling.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_comprehendread_retry_strategy.ts" + ], + "promptPartDocCommentCount": 15, + "promptDocCommentCount": 0, + "benchmarkDefinitionCount": 15, + "intentMetadataCount": 15, + "currentVersionMetadataCount": 15, + "versionsMetadataCount": 15, + "promptPartExportCount": 15, + "promptConstructionCount": 0, + "promptSetCallCount": 0, + "promptRequireCallCount": 0, + "promptRequiresHierarchyCount": 0, + "extendsPromptCount": 0, + "extendsDocCodeToolPromptCount": 0, + "docCodeToolPromptCount": 0, + "formatUsableToolsCount": 0, + "readNeedComprehensionNamesCount": 7, + "readFitsFindingNamesCount": 1, + "depositorySearchToolNamesCount": 0, + "promptTemplateIdCount": 1, + "benchmarkRunnerPromptPartMethodsCount": 0, + "benchmarkRunnerPromptMethodsCount": 0, + "promptPartSuiteTypeCount": 0, + "promptSuiteTypeCount": 0, + "extractBenchmarksCount": 0, + "pbvDocumentationCount": 0 + }, + "promptpart:read-fits-finding-specific": { + "rowId": "promptpart:read-fits-finding-specific", + "sourceRootExists": true, + "sourceFileCount": 201, + "sourceFileSample": [ + "packages/pipelines/asset-pack/src/depository-search.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_assetpackdiscoveryanalyzeparallel_capabilities_list.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_assetpackdiscoveryanalyzeparallel_identity_definition.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_assetpackdiscoveryanalyzeparallel_ptrrplan_purpose.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_assetpackdiscoveryanalyzeparallel_ptrrrefine_purpose.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_assetpackdiscoveryanalyzeparallel_ptrrretry_purpose.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_assetpackdiscoveryanalyzeparallel_ptrrtry_purpose.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_assetpackdiscoveryanalyzeparallel_purpose_corestatement.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_assetpackdiscoveryanalyzeparallel_requirements_context.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_assetpackdiscoveryanalyzeparallel_tools_available.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_assetpackdiscoveryassesscomplexity_capabilities_list.ts", + "packages/prompts/src/raw_promptparts/specific/promptpart_specific_agent_assetpackdiscoveryassesscomplexity_identity_definition.ts" + ], + "promptPartDocCommentCount": 337, + "promptDocCommentCount": 0, + "benchmarkDefinitionCount": 337, + "intentMetadataCount": 337, + "currentVersionMetadataCount": 337, + "versionsMetadataCount": 337, + "promptPartExportCount": 200, + "promptConstructionCount": 0, + "promptSetCallCount": 1, + "promptRequireCallCount": 0, + "promptRequiresHierarchyCount": 0, + "extendsPromptCount": 0, + "extendsDocCodeToolPromptCount": 0, + "docCodeToolPromptCount": 0, + "formatUsableToolsCount": 0, + "readNeedComprehensionNamesCount": 0, + "readFitsFindingNamesCount": 8, + "depositorySearchToolNamesCount": 6, + "promptTemplateIdCount": 0, + "benchmarkRunnerPromptPartMethodsCount": 0, + "benchmarkRunnerPromptMethodsCount": 0, + "promptPartSuiteTypeCount": 0, + "promptSuiteTypeCount": 0, + "extractBenchmarksCount": 0, + "pbvDocumentationCount": 0 + }, + "prompt:reading-pipeline-agent-prompts": { + "rowId": "prompt:reading-pipeline-agent-prompts", + "sourceRootExists": true, + "sourceFileCount": 18, + "sourceFileSample": [ + "packages/pipelines/asset-pack/src/agents/prompts/analyze-codebase-prompt.ts", + "packages/pipelines/asset-pack/src/agents/prompts/analyze-parallel-prompt.ts", + "packages/pipelines/asset-pack/src/agents/prompts/assess-complexity-prompt.ts", + "packages/pipelines/asset-pack/src/agents/prompts/asset-pack-comprehend-read-agent-prompts.ts", + "packages/pipelines/asset-pack/src/agents/prompts/asset-pack-validation-ready-to-finish-prompt.ts", + "packages/pipelines/asset-pack/src/agents/prompts/asset-pack-vcs-clone-repository-agent-prompts.ts", + "packages/pipelines/asset-pack/src/agents/prompts/clone-vcs-repository-prompt.ts", + "packages/pipelines/asset-pack/src/agents/prompts/comprehend-attachments-prompt.ts", + "packages/pipelines/asset-pack/src/agents/prompts/comprehend-read-prompt.ts", + "packages/pipelines/asset-pack/src/agents/prompts/create-pull-request-prompt.ts", + "packages/pipelines/asset-pack/src/agents/prompts/danger-wall-prompt.ts", + "packages/pipelines/asset-pack/src/agents/prompts/finalize-delivery-evidence-prompt.ts" + ], + "promptPartDocCommentCount": 0, + "promptDocCommentCount": 21, + "benchmarkDefinitionCount": 21, + "intentMetadataCount": 21, + "currentVersionMetadataCount": 21, + "versionsMetadataCount": 0, + "promptPartExportCount": 0, + "promptConstructionCount": 85, + "promptSetCallCount": 764, + "promptRequireCallCount": 46, + "promptRequiresHierarchyCount": 0, + "extendsPromptCount": 0, + "extendsDocCodeToolPromptCount": 0, + "docCodeToolPromptCount": 0, + "formatUsableToolsCount": 0, + "readNeedComprehensionNamesCount": 11, + "readFitsFindingNamesCount": 26, + "depositorySearchToolNamesCount": 2, + "promptTemplateIdCount": 6, + "benchmarkRunnerPromptPartMethodsCount": 0, + "benchmarkRunnerPromptMethodsCount": 0, + "promptPartSuiteTypeCount": 0, + "promptSuiteTypeCount": 0, + "extractBenchmarksCount": 0, + "pbvDocumentationCount": 0 + }, + "prompt:conversation-system-prompt": { + "rowId": "prompt:conversation-system-prompt", + "sourceRootExists": true, + "sourceFileCount": 1, + "sourceFileSample": [ + "packages/conversations-generics/src/prompts/BitcodeTerminalConversationSystemPrompt.ts" + ], + "promptPartDocCommentCount": 0, + "promptDocCommentCount": 1, + "benchmarkDefinitionCount": 1, + "intentMetadataCount": 1, + "currentVersionMetadataCount": 1, + "versionsMetadataCount": 1, + "promptPartExportCount": 0, + "promptConstructionCount": 0, + "promptSetCallCount": 3, + "promptRequireCallCount": 3, + "promptRequiresHierarchyCount": 1, + "extendsPromptCount": 1, + "extendsDocCodeToolPromptCount": 0, + "docCodeToolPromptCount": 0, + "formatUsableToolsCount": 0, + "readNeedComprehensionNamesCount": 0, + "readFitsFindingNamesCount": 0, + "depositorySearchToolNamesCount": 0, + "promptTemplateIdCount": 0, + "benchmarkRunnerPromptPartMethodsCount": 0, + "benchmarkRunnerPromptMethodsCount": 0, + "promptPartSuiteTypeCount": 0, + "promptSuiteTypeCount": 0, + "extractBenchmarksCount": 0, + "pbvDocumentationCount": 0 + }, + "prompt:tool-definition-doc-code-prompts": { + "rowId": "prompt:tool-definition-doc-code-prompts", + "sourceRootExists": true, + "sourceFileCount": 14, + "sourceFileSample": [ + "packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.ts", + "packages/generic-tools/vcs/src/prompts/CreateCommentDocCodeToolPrompt.ts", + "packages/generic-tools/vcs/src/prompts/CreateIssueDocCodeToolPrompt.ts", + "packages/generic-tools/vcs/src/prompts/CreateOrUpdateFileDocCodeToolPrompt.ts", + "packages/generic-tools/vcs/src/prompts/CreatePullRequestDocCodeToolPrompt.ts", + "packages/generic-tools/vcs/src/prompts/GetFileContentDocCodeToolPrompt.ts", + "packages/generic-tools/vcs/src/prompts/ListBranchesDocCodeToolPrompt.ts", + "packages/generic-tools/vcs/src/prompts/ListRepositoriesDocCodeToolPrompt.ts", + "packages/generic-tools/vcs/src/prompts/index.ts", + "packages/generic-tools/web-search/src/prompts/GetContentsDocCodeToolPrompt.ts", + "packages/generic-tools/web-search/src/prompts/MultiProviderSearchDocCodeToolPrompt.ts", + "packages/generic-tools/web-search/src/prompts/WebSearchDocCodeToolPrompt.ts" + ], + "promptPartDocCommentCount": 0, + "promptDocCommentCount": 13, + "benchmarkDefinitionCount": 2, + "intentMetadataCount": 13, + "currentVersionMetadataCount": 2, + "versionsMetadataCount": 12, + "promptPartExportCount": 0, + "promptConstructionCount": 0, + "promptSetCallCount": 164, + "promptRequireCallCount": 0, + "promptRequiresHierarchyCount": 0, + "extendsPromptCount": 1, + "extendsDocCodeToolPromptCount": 11, + "docCodeToolPromptCount": 87, + "formatUsableToolsCount": 1, + "readNeedComprehensionNamesCount": 0, + "readFitsFindingNamesCount": 0, + "depositorySearchToolNamesCount": 0, + "promptTemplateIdCount": 0, + "benchmarkRunnerPromptPartMethodsCount": 0, + "benchmarkRunnerPromptMethodsCount": 0, + "promptPartSuiteTypeCount": 0, + "promptSuiteTypeCount": 0, + "extractBenchmarksCount": 0, + "pbvDocumentationCount": 0 + } + }, + "coverage": { + "rowCount": 7, + "subjectKindIds": [ + "benchmark-infrastructure", + "PromptPart", + "Prompt", + "DocCodeToolPrompt" + ], + "metricIds": [ + "intent_alignment", + "semantic_clarity", + "token_efficiency", + "model_stability", + "task_success", + "response_quality", + "schema_conformance", + "source_boundary_preservation" + ], + "disclosureTierIds": [ + "prompt_identity_source_safe", + "promptpart_identity_source_safe", + "benchmark_fixture_source_safe", + "expected_typed_output_quality_source_safe", + "benchmark_score_source_safe", + "raw_prompt_text_private", + "raw_provider_response_private" + ], + "promptPartSuiteCount": 3, + "completePromptSuiteCount": 3, + "benchmarkInfrastructureSuiteCount": 1, + "fixtureCount": 13, + "expectedTypedOutputQualityCount": 24, + "requiredPredicateCount": 38, + "passedPredicateCount": 38, + "failedPredicateIds": [], + "sourceRootCount": 16, + "missingSourceRoots": [], + "legacySourceRoots": false, + "sourceFileCount": 313, + "promptPartDocCommentCount": 443, + "promptDocCommentCount": 39, + "benchmarkDefinitionCount": 465, + "promptPartExportCount": 275, + "promptConstructionCount": 85, + "gate2InventoryRoot": "v38-inference-surface-inventory:3f50930549dee560db576a9a", + "gate3StackRoot": "v38-ptrr-failsafe-thricified-stack:af3a75b196d4b52e921c0130", + "sourceSafeMetadataOnly": true, + "knownCarryForwardGapIds": [ + "gate5-streams-benchmark-linked-inference-telemetry-with-disclosure-tiers", + "gate6-hardens-read-need-comprehension-prompt-return-types", + "gate7-hardens-read-fits-finding-query-prompt-benchmarking", + "gate9-applies-benchmark-report-to-conversation-tool-prompt-parity" + ] + }, + "failures": [], + "passed": true, + "artifactRoot": "v38-prompt-benchmark-report:7c34a5c11c25c490aedf413d" +} diff --git a/.github/workflows/bitcode-canon-quality.yml b/.github/workflows/bitcode-canon-quality.yml index 23b724e13..fb0974e8a 100644 --- a/.github/workflows/bitcode-canon-quality.yml +++ b/.github/workflows/bitcode-canon-quality.yml @@ -151,6 +151,9 @@ jobs: if [ -f scripts/check-v38-gate3-ptrr-failsafe-thricified-stack.mjs ]; then node scripts/check-v38-gate3-ptrr-failsafe-thricified-stack.mjs --skip-branch-check fi + if [ -f scripts/check-v38-gate4-prompt-benchmark-report.mjs ]; then + node scripts/check-v38-gate4-prompt-benchmark-report.mjs --skip-branch-check + fi fi else echo "Unexpected BITCODE_SPEC.txt pointer: $POINTER" >&2 diff --git a/.github/workflows/bitcode-gate-quality.yml b/.github/workflows/bitcode-gate-quality.yml index fbb626e01..f2da31209 100644 --- a/.github/workflows/bitcode-gate-quality.yml +++ b/.github/workflows/bitcode-gate-quality.yml @@ -280,6 +280,9 @@ jobs: if [ -f scripts/check-v38-gate3-ptrr-failsafe-thricified-stack.mjs ]; then node scripts/check-v38-gate3-ptrr-failsafe-thricified-stack.mjs --skip-branch-check fi + if [ -f scripts/check-v38-gate4-prompt-benchmark-report.mjs ]; then + node scripts/check-v38-gate4-prompt-benchmark-report.mjs --skip-branch-check + fi fi else echo "Unexpected BITCODE_SPEC.txt pointer: $POINTER" >&2 @@ -362,6 +365,9 @@ jobs: if [ -f packages/protocol/test/v38-ptrr-failsafe-thricified-stack.test.js ]; then pnpm --dir packages/protocol exec node --test --test-force-exit test/v38-ptrr-failsafe-thricified-stack.test.js fi + if [ -f packages/protocol/test/v38-prompt-benchmark-report.test.js ]; then + pnpm --dir packages/protocol exec node --test --test-force-exit test/v38-prompt-benchmark-report.test.js + fi pnpm --filter @bitcode/protocol test - name: Test staged Reading route and Terminal harness contracts diff --git a/BITCODE_SPEC_V38.md b/BITCODE_SPEC_V38.md index 58a5fb2df..98a0a406e 100644 --- a/BITCODE_SPEC_V38.md +++ b/BITCODE_SPEC_V38.md @@ -8,7 +8,7 @@ - Prior canonical anchor: `BITCODE_SPEC_V37.md` - Prior generated proof appendix: `BITCODE_SPEC_V37_PROVEN.md` - Active canonical pointer during draft opening: `BITCODE_SPEC.txt` -> `V37` -- Generated structured artifact inventory: draft `.bitcode/v38-spec-family-report.json`, `.bitcode/v38-canonical-input-report.json`, `.bitcode/v38-canon-posture-drift-report.json`, `.bitcode/v38-inference-surface-inventory.json`, `.bitcode/v38-ptrr-failsafe-thricified-stack.json`, V38 gate-quality workflow evidence, and future V38 generated proof artifacts as gates close +- Generated structured artifact inventory: draft `.bitcode/v38-spec-family-report.json`, `.bitcode/v38-canonical-input-report.json`, `.bitcode/v38-canon-posture-drift-report.json`, `.bitcode/v38-inference-surface-inventory.json`, `.bitcode/v38-ptrr-failsafe-thricified-stack.json`, `.bitcode/v38-prompt-benchmark-report.json`, V38 gate-quality workflow evidence, and future V38 generated proof artifacts as gates close - Source parity state: V38 source-side inference stack, prompt benchmarking, Reading pipeline, depository-search, telemetry, rehearsal, workflow, and promotion surfaces are draft-required until their gates close - Notes companion: `BITCODE_SPEC_V38_NOTES.md` - Delta companion: `BITCODE_SPEC_V38_DELTA.md` @@ -86,8 +86,11 @@ Gate 3 confirms tools remain step-owned postprocess capabilities while `Failsafe ### Gate 4: PromptPart And Prompt Benchmarking Gate 4 completes prompt and PromptPart benchmarking. +Prompt benchmarking is the source-safe measurement contract for complete Prompt registries and their composed PromptParts. Semantically divided PromptParts and complete Prompts must be benchmarkable with initial benchmark suites for active Reading, Conversation, and tool-definition inference surfaces. Benchmarks must preserve prompt identity, fixture inputs, expected typed-output qualities, disclosure tiers, and stable report artifacts. +Gate 4 is closed by the package-backed `V38PromptBenchmarkReport` source and deterministic `.bitcode/v38-prompt-benchmark-report.json` artifact. +The artifact has `source-safe-prompt-benchmark-metadata` disclosure posture and records 7 benchmark rows, 13 source-safe fixtures, 24 typed-output quality expectations, 38 passed source predicates, 443 active PromptPart doc-comments, 39 complete Prompt doc-comments, 465 benchmark definitions, 275 PromptPart exports, and 85 Prompt constructions across Reading, Conversation, and tool-definition inference surfaces without serializing raw prompt text, protected source, credentials, raw provider responses, or unpaid AssetPack source. ### Gate 5: Inference Telemetry And Disclosure Tiers diff --git a/BITCODE_SPEC_V38_DELTA.md b/BITCODE_SPEC_V38_DELTA.md index f530453d0..93622b4c9 100644 --- a/BITCODE_SPEC_V38_DELTA.md +++ b/BITCODE_SPEC_V38_DELTA.md @@ -7,7 +7,7 @@ - Current canonical/latest target: `V37` - Prior canonical anchor: `BITCODE_SPEC_V37.md` - Prior generated proof appendix: `BITCODE_SPEC_V37_PROVEN.md` -- Generated structured artifact inventory: draft `.bitcode/v38-spec-family-report.json`, `.bitcode/v38-canonical-input-report.json`, `.bitcode/v38-canon-posture-drift-report.json`, `.bitcode/v38-inference-surface-inventory.json`, `.bitcode/v38-ptrr-failsafe-thricified-stack.json`, V38 gate-quality workflow evidence, and future V38 generated proof artifacts as gates close +- Generated structured artifact inventory: draft `.bitcode/v38-spec-family-report.json`, `.bitcode/v38-canonical-input-report.json`, `.bitcode/v38-canon-posture-drift-report.json`, `.bitcode/v38-inference-surface-inventory.json`, `.bitcode/v38-ptrr-failsafe-thricified-stack.json`, `.bitcode/v38-prompt-benchmark-report.json`, V38 gate-quality workflow evidence, and future V38 generated proof artifacts as gates close - Source parity state: V38 source-side inference stack, prompt benchmarking, Reading pipeline, depository-search, telemetry, rehearsal, workflow, and promotion surfaces are draft-required until their gates close - Spec companion: `BITCODE_SPEC_V38.md` - Notes companion: `BITCODE_SPEC_V38_NOTES.md` @@ -105,10 +105,17 @@ Closure implementation: Closure acceptance: - PromptParts and complete Prompts are benchmarkable; +- Prompt benchmarking remains source-safe and registry-backed; - active Reading, Conversation, and tool-definition prompts have initial benchmark suites; - benchmark artifacts preserve prompt identities, fixtures, typed-output quality expectations, and disclosure tiers; - `pnpm run check:v38-gate4` validates benchmark runner, fixtures, reports, and source-safe metadata. +Closure implementation: + +- `V38PromptBenchmarkReport` is now package-backed in `packages/protocol/src/canonical/prompt-benchmark-report.js` and generated to `.bitcode/v38-prompt-benchmark-report.json`. +- The generated artifact is `source-safe-prompt-benchmark-metadata` and covers benchmark infrastructure, generic PTRR/Failsafe/ThricifiedGeneration PromptParts, `ReadNeedComprehensionSynthesis` PromptParts, `ReadFitsFindingSynthesis` PromptParts, complete Reading Prompt registries, Website Conversation Prompts, and DocCodeToolPrompt surfaces. +- The current benchmark report records 7 rows, 13 source-safe fixtures, 24 typed-output quality expectations, 38 passed source predicates, 443 active PromptPart doc-comments, 39 complete Prompt doc-comments, 465 benchmark definitions, 275 PromptPart exports, and 85 Prompt constructions without serializing raw prompt text, protected source, raw provider responses, credentials, or unpaid AssetPack source. + ### Gate 5: Inference Telemetry And Disclosure Tiers Closure acceptance: diff --git a/BITCODE_SPEC_V38_NOTES.md b/BITCODE_SPEC_V38_NOTES.md index 3a3445c4c..3580267bb 100644 --- a/BITCODE_SPEC_V38_NOTES.md +++ b/BITCODE_SPEC_V38_NOTES.md @@ -8,7 +8,7 @@ - Current active draft target: `V38` - Prior canonical anchor: `BITCODE_SPEC_V37.md` - Prior generated proof appendix: `BITCODE_SPEC_V37_PROVEN.md` -- Generated structured artifact inventory: draft `.bitcode/v38-spec-family-report.json`, `.bitcode/v38-canonical-input-report.json`, `.bitcode/v38-canon-posture-drift-report.json`, `.bitcode/v38-inference-surface-inventory.json`, `.bitcode/v38-ptrr-failsafe-thricified-stack.json`, V38 gate-quality workflow evidence, and future V38 generated proof artifacts as gates close +- Generated structured artifact inventory: draft `.bitcode/v38-spec-family-report.json`, `.bitcode/v38-canonical-input-report.json`, `.bitcode/v38-canon-posture-drift-report.json`, `.bitcode/v38-inference-surface-inventory.json`, `.bitcode/v38-ptrr-failsafe-thricified-stack.json`, `.bitcode/v38-prompt-benchmark-report.json`, V38 gate-quality workflow evidence, and future V38 generated proof artifacts as gates close - Source parity state: V38 source-side inference stack, prompt benchmarking, Reading pipeline, depository-search, telemetry, rehearsal, workflow, and promotion surfaces are draft-required until their gates close - Spec companion: `BITCODE_SPEC_V38.md` - Delta companion: `BITCODE_SPEC_V38_DELTA.md` @@ -162,6 +162,15 @@ Gate 3 closes the practical PTRR stack contract by making the existing source-ba The artifact proves 69 source predicates and binds 52 PTRR steps to 156 Failsafe sequences, 156 ThricifiedGeneration chains, and 468 provider-call slots. Tools remain step-owned postprocess executions; `ThricifiedGeneration` remains the lowest-level Reason, Judge, StructuredOutput provider-call chain. +## Gate 4 implementation notes + +Gate 4 closes the first concrete prompt benchmarking contract by making benchmark coverage package-backed, generated, tested, and workflow-checkable. +Prompt benchmarking remains the complete-Prompt side of the same benchmark law that applies to semantically divided PromptParts. +`V38PromptBenchmarkReport` and `.bitcode/v38-prompt-benchmark-report.json` record prompt benchmark infrastructure, generic PTRR/Failsafe/ThricifiedGeneration PromptParts, `ReadNeedComprehensionSynthesis` PromptParts, `ReadFitsFindingSynthesis` PromptParts, complete Reading Prompt registries, Website Conversation Prompts, and DocCodeToolPrompt surfaces as `source-safe-prompt-benchmark-metadata`. +The artifact deliberately stores prompt identities, source-safe fixture ids, typed-output quality expectations, disclosure tiers, source roots, counts, predicate ids, and proof roots rather than raw prompt text or provider responses. +The current generated report covers 7 benchmark rows, 13 fixtures, 24 typed-output quality expectations, 38 source predicates, 443 active PromptPart doc-comments, 39 complete Prompt doc-comments, 465 benchmark definitions, 275 PromptPart exports, and 85 Prompt constructions. +Later V38 gates must use this report when streaming inference telemetry, hardening the two Reading pipelines, and applying Conversation/tool-prompt parity. + ## V38 gate plan - Gate 1: V38 Inference Stack Roadmap And Spec Opening diff --git a/BITCODE_SPEC_V38_PARITY_MATRIX.md b/BITCODE_SPEC_V38_PARITY_MATRIX.md index e7b775135..85c0e0822 100644 --- a/BITCODE_SPEC_V38_PARITY_MATRIX.md +++ b/BITCODE_SPEC_V38_PARITY_MATRIX.md @@ -7,7 +7,7 @@ - Current canonical/latest target: `V37` - Prior canonical anchor: `BITCODE_SPEC_V37.md` - Prior generated proof appendix: `BITCODE_SPEC_V37_PROVEN.md` -- Generated structured artifact inventory: draft `.bitcode/v38-spec-family-report.json`, `.bitcode/v38-canonical-input-report.json`, `.bitcode/v38-canon-posture-drift-report.json`, `.bitcode/v38-inference-surface-inventory.json`, `.bitcode/v38-ptrr-failsafe-thricified-stack.json`, V38 gate-quality workflow evidence, and future V38 generated proof artifacts as gates close +- Generated structured artifact inventory: draft `.bitcode/v38-spec-family-report.json`, `.bitcode/v38-canonical-input-report.json`, `.bitcode/v38-canon-posture-drift-report.json`, `.bitcode/v38-inference-surface-inventory.json`, `.bitcode/v38-ptrr-failsafe-thricified-stack.json`, `.bitcode/v38-prompt-benchmark-report.json`, V38 gate-quality workflow evidence, and future V38 generated proof artifacts as gates close - Source parity state: V38 source-side inference stack, prompt benchmarking, Reading pipeline, depository-search, telemetry, rehearsal, workflow, and promotion surfaces are draft-required until their gates close - Spec companion: `BITCODE_SPEC_V38.md` - Notes companion: `BITCODE_SPEC_V38_NOTES.md` @@ -64,6 +64,11 @@ Gate 1 audit basis: - `packages/protocol/test/v38-ptrr-failsafe-thricified-stack.test.js` - `scripts/generate-v38-ptrr-failsafe-thricified-stack.mjs` - `scripts/check-v38-gate3-ptrr-failsafe-thricified-stack.mjs` +- `.bitcode/v38-prompt-benchmark-report.json` +- `packages/protocol/src/canonical/prompt-benchmark-report.js` +- `packages/protocol/test/v38-prompt-benchmark-report.test.js` +- `scripts/generate-v38-prompt-benchmark-report.mjs` +- `scripts/check-v38-gate4-prompt-benchmark-report.mjs` No `_legacy/` source is active source truth. @@ -80,6 +85,7 @@ No `_legacy/` source is active source truth. | Telemetry disclosure scope | Gate 1 | V35 telemetry law, V37 stream UI law, V38 inference notes | drafted | Gate plan requires source-safe prompt, raw response, parsed output, schema verdict, retry, and repair visibility at permitted tiers. | | Inference surface inventory | Gate 2 | `V38InferenceSurfaceInventory`, `.bitcode/v38-inference-surface-inventory.json`, protocol test, generator, checker, workflows | closed | Source-safe inventory covers Reading, Conversation, tool-definition prompt, interface entrypoint, prompt registry, and execution primitive families with 52 PTRR steps and later-gate gaps explicit. | | PTRR Failsafe and Thricified execution stack | Gate 3 | `V38PtrrFailsafeThricifiedStack`, `.bitcode/v38-ptrr-failsafe-thricified-stack.json`, protocol test, generator, checker, workflows | closed | Practical PTRR agents prove four steps, three Failsafe stages, three ThricifiedGeneration stages, 69 source predicates, and Gate 2 count binding with step-owned tools. | +| PromptPart and Prompt benchmarking | Gate 4 | `V38PromptBenchmarkReport`, `.bitcode/v38-prompt-benchmark-report.json`, protocol test, generator, checker, workflows | closed | Source-safe benchmark report covers active Reading, Conversation, and tool-definition PromptParts and complete Prompts with 7 rows, 13 fixtures, 24 typed-output quality expectations, and 38 passed source predicates. | ## V38 implementation checklist @@ -94,6 +100,7 @@ No `_legacy/` source is active source truth. | Package docs | README, protocol package README, demonstration README, and PR template state V37 active / V38 draft workflow | drafted | | Inference surface inventory | `.bitcode/v38-inference-surface-inventory.json` and `V38InferenceSurfaceInventory` are generated, tested, checked, documented, and workflow-wired as `source-safe-inference-surface-metadata` | closed | | PTRR stack artifact | `.bitcode/v38-ptrr-failsafe-thricified-stack.json` and `V38PtrrFailsafeThricifiedStack` are generated, tested, checked, documented, and workflow-wired as `source-safe-ptrr-failsafe-thricified-stack-metadata` | closed | +| Prompt benchmark artifact | `.bitcode/v38-prompt-benchmark-report.json` and `V38PromptBenchmarkReport` are generated, tested, checked, documented, and workflow-wired as `source-safe-prompt-benchmark-metadata` | closed | | Inference stack vocabulary | V38 spec family names `PipelineExecution`, PTRR agents, Plan, Try, Refine, Retry, `FailsafeGenerationSequence`, `ThricifiedGeneration`, `ToolExecution`, `DocCodeToolPrompt`, and provider call boundaries | drafted | | Reading vocabulary | V38 spec family names `ReadNeedComprehensionSynthesis` and `ReadFitsFindingSynthesis` | drafted | | Depository search vocabulary | V38 spec family names lexical, symbolic, path, metadata, measurement, embedding/vector, provider-specific channels, candidate deposits, ranking, thresholds, and selected-fit provenance | drafted | @@ -146,9 +153,25 @@ No `_legacy/` source is active source truth. | Gate 2 counts are bound | 52 PTRR steps, 156 Failsafe sequences, 156 ThricifiedGeneration chains, 468 provider-call slots | closed | | Gate checker is wired | `pnpm run check:v38-gate3`, package test, generator check, gate/canon workflows | closed | +## Gate 4 Parity + +| Requirement | Source evidence | Current V38 judgment | +| --- | --- | --- | +| Benchmark report source is package-owned | `packages/protocol/src/canonical/prompt-benchmark-report.js`, `packages/protocol/src/index.js`, `packages/protocol/src/index.d.ts` | closed | +| Deterministic artifact exists | `.bitcode/v38-prompt-benchmark-report.json` | closed | +| Source-safe disclosure posture is explicit | `source-safe-prompt-benchmark-metadata`, fixture ids, typed-output quality ids, raw prompt text private, raw provider response private | closed | +| Benchmark runner and suite types are included | `packages/prompts/src/benchmarking/runner.ts`, `packages/prompts/src/benchmarking/types.ts`, `packages/prompts/src/developing/doc-comment-developing.ts` | closed | +| Generic PromptParts are benchmarkable | generic PTRR, Failsafe, ThricifiedGeneration, formatting, and DocCode PromptParts | closed | +| Reading PromptParts are benchmarkable | `ReadNeedComprehensionSynthesis` and `ReadFitsFindingSynthesis` specific PromptParts and typed-output quality fixtures | closed | +| Complete Reading Prompts are benchmarkable | `packages/pipelines/asset-pack/src/agents/prompts`, `packages/pipelines/asset-pack/src/reading-pipeline-contract.ts` | closed | +| Conversation Prompt is benchmarkable | `BitcodeTerminalConversationSystemPrompt` benchmark row with route-local source-safe fixture | closed | +| Tool-definition Prompts are benchmarkable | `DocCodeToolPrompt`, `formatUsableTools`, web-search, repository-evidence search, and VCS PR tool prompts | closed | +| Count contract is explicit | 7 rows, 13 fixtures, 24 typed-output quality expectations, 38 source predicates, 443 PromptPart doc-comments, 39 complete Prompt doc-comments, 465 benchmark definitions, 275 PromptPart exports, and 85 Prompt constructions | closed | +| Gate checker is wired | `pnpm run check:v38-gate4`, package test, generator check, gate/canon workflows | closed | + ## Completion condition -Gate 3 closes when `pnpm run check:v38-gate3`, the V38 PTRR/Failsafe/Thricified stack generator check, Gate 2 inventory checks, V38 draft spec-family validation over V37, V37/V38 canon-posture drift validation, canonical input validation for V37, strict V38 spec quality, protocol tests, workflow wiring checks, secret scans, and diff hygiene all pass on `v38/gate-3-ptrr-failsafe-thricified-stack`. +Gate 4 closes when `pnpm run check:v38-gate4`, the V38 prompt benchmark report generator check, Gate 3 PTRR/Failsafe/Thricified stack checks, Gate 2 inventory checks, V38 draft spec-family validation over V37, V37/V38 canon-posture drift validation, canonical input validation for V37, strict V38 spec quality, protocol tests, workflow wiring checks, secret scans, and diff hygiene all pass on `v38/gate-4-promptpart-prompt-benchmarking`. ## accepted boundaries diff --git a/README.md b/README.md index 9f65471ea..4eaa255c2 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,21 @@ slots as `source-safe-ptrr-failsafe-thricified-stack-metadata`. Use `pnpm run generate:v38-ptrr-failsafe-thricified-stack`, `pnpm run check:v38-ptrr-failsafe-thricified-stack`, and `pnpm run check:v38-gate3` before closing the gate. +V38 Gate 4 adds the package-backed `V38PromptBenchmarkReport` and generated +source-safe artifact `.bitcode/v38-prompt-benchmark-report.json`. The Prompt +benchmarking report covers benchmark infrastructure, generic PTRR/Failsafe/ +ThricifiedGeneration PromptParts, `ReadNeedComprehensionSynthesis` +PromptParts, `ReadFitsFindingSynthesis` PromptParts, complete Reading Prompt +registries, Website Conversation Prompts, and DocCodeToolPrompt surfaces as +`source-safe-prompt-benchmark-metadata`. The current source-safe count contract +is 7 rows, 13 fixtures, 24 typed-output quality expectations, 38 source +predicates, 443 PromptPart doc-comments, 39 complete Prompt doc-comments, 465 +benchmark definitions, 275 PromptPart exports, and 85 Prompt constructions. +Prompt benchmarking artifacts never serialize raw prompt text or raw provider +responses. +Use `pnpm run generate:v38-prompt-benchmark-report`, +`pnpm run check:v38-prompt-benchmark-report`, and `pnpm run check:v38-gate4` +before closing the gate. V36 Gate 2 anchors market-wide activity through the package-owned `ExchangeActivityBook` and the source-safe generated artifact `.bitcode/v36-exchange-activity-book.json`, including listing, bid, ask, @@ -285,8 +300,9 @@ inference stack, Reading pipeline, depository-search, prompt benchmark, telemetry, and rehearsal posture, generate `BITCODE_SPEC_V38_PROVEN.md`, and commit promotion artifacts plus the `BITCODE_SPEC.txt` pointer change from `V37` to `V38` on the version branch. -V38 Gates 1 through 3 are wired through `pnpm run check:v38-gate1`, -`pnpm run check:v38-gate2`, and `pnpm run check:v38-gate3`. The promoted V37 +V38 Gates 1 through 4 are wired through `pnpm run check:v38-gate1`, +`pnpm run check:v38-gate2`, `pnpm run check:v38-gate3`, and +`pnpm run check:v38-gate4`. The promoted V37 gate closure remains reproducible through `pnpm run check:v37-gate1`; V37 Gate 2 is wired through `pnpm run check:v37-gate2` and diff --git a/SPECIFICATIONS_ROADMAP.md b/SPECIFICATIONS_ROADMAP.md index efcc69750..e9d8236b3 100644 --- a/SPECIFICATIONS_ROADMAP.md +++ b/SPECIFICATIONS_ROADMAP.md @@ -5,8 +5,8 @@ - Current active canonical pointer: `BITCODE_SPEC.txt` -> `V37` - Current active canon: `BITCODE_SPEC_V37.md` - Current draft target: `BITCODE_SPEC_V38.md`. -- Current working gate: V38 Gate 4 PromptPart And Prompt Benchmarking. -- Next queued gate after V38 Gate 3 closure: V38 Gate 4 PromptPart And Prompt Benchmarking. +- Current working gate: V38 Gate 5 Inference Telemetry And Disclosure Tiers. +- Next queued gate after V38 Gate 4 closure: V38 Gate 5 Inference Telemetry And Disclosure Tiers. - Latest closed version: V37 Website Conversations, which promoted conversation sessions, route-local history, stream UI/event contracts, fullscreen writing, source selectors, Terminal handoff, persistence/privacy/redaction, telemetry/proof hooks, local/staging rehearsal, and V37 promotion readiness. - Recent V37 closure anchor: V37 Gate 10 Promotion Readiness generated V37 proof support, promoted `BITCODE_SPEC.txt` to `V37`, prepared V37 active / V38 draft runtime posture, and closed Website Conversations canon. - V34 Gate 2 closure anchor: deployment-depth now owns package-backed `DeploymentHostCapabilityCatalog` and `EnvironmentLaneContract` source, deterministic `.bitcode/v34-deployment-host-capability-catalog.json` and `.bitcode/v34-environment-lane-contracts.json`, and visible `value-bearing-mainnet` blocking through `blocked_future_canon_required`. @@ -52,6 +52,7 @@ - V38 Gate 1 opening anchor: Inference correctness depth opens over promoted V37 with V38 SPEC, DELTA, NOTES, and PARITY files, `check:v38-gate1`, V37 active / V38 draft posture, and an eleven-gate plan for inference surface inventory, PTRR Failsafe over Thricified execution stack, prompt and PromptPart benchmarking, inference telemetry and disclosure tiers, ReadNeedComprehensionSynthesis hardening, ReadFitsFindingSynthesis depository search and embeddings, AssetPack synthesis handoff and economic traceability, Conversation/tool-prompt parity, local/staging inference rehearsal, and promotion readiness. - V38 Gate 2 closure anchor: Inference correctness depth now owns package-backed `V38InferenceSurfaceInventory` source, deterministic `.bitcode/v38-inference-surface-inventory.json`, `source-safe-inference-surface-metadata`, protocol test, generator/checker scripts, workflow wiring, and 52 PTRR steps / 156 Failsafe+Thricified chains / 468 provider-call slots across Reading, Conversation, tool-definition prompt, interface entrypoint, prompt registry, and execution primitive surfaces. - V38 Gate 3 closure anchor: Inference correctness depth now owns package-backed `V38PtrrFailsafeThricifiedStack` source, deterministic `.bitcode/v38-ptrr-failsafe-thricified-stack.json`, `source-safe-ptrr-failsafe-thricified-stack-metadata`, protocol test, generator/checker scripts, workflow wiring, 69 passed source predicates, all four PTRR steps, all three Failsafe stages, all three ThricifiedGeneration stages, step-owned tools, and Gate 2's 52 PTRR steps / 156 Failsafe sequences / 156 ThricifiedGeneration chains / 468 provider-call slots bound into the practical call-stack law. +- V38 Gate 4 closure anchor: Inference correctness depth now owns package-backed `V38PromptBenchmarkReport` source, deterministic `.bitcode/v38-prompt-benchmark-report.json`, `source-safe-prompt-benchmark-metadata`, protocol test, generator/checker scripts, workflow wiring, 7 benchmark rows, 13 source-safe fixtures, 24 typed-output quality expectations, 38 passed source predicates, 443 active PromptPart doc-comments, 39 complete Prompt doc-comments, 465 benchmark definitions, 275 PromptPart exports, and 85 Prompt constructions across active Reading, Conversation, and tool-definition inference surfaces. - Purpose: concise running index of Bitcode/ENGI specification history, current work, and planned work. This roadmap is not an active system specification. diff --git a/package.json b/package.json index bf0c29151..67fd1b040 100644 --- a/package.json +++ b/package.json @@ -220,6 +220,9 @@ "generate:v38-ptrr-failsafe-thricified-stack": "node scripts/generate-v38-ptrr-failsafe-thricified-stack.mjs", "check:v38-ptrr-failsafe-thricified-stack": "node scripts/generate-v38-ptrr-failsafe-thricified-stack.mjs --check", "check:v38-gate3": "node scripts/check-v38-gate3-ptrr-failsafe-thricified-stack.mjs", + "generate:v38-prompt-benchmark-report": "node scripts/generate-v38-prompt-benchmark-report.mjs", + "check:v38-prompt-benchmark-report": "node scripts/generate-v38-prompt-benchmark-report.mjs --check", + "check:v38-gate4": "node scripts/check-v38-gate4-prompt-benchmark-report.mjs", "check:spec-quality": "node scripts/run-bitcode-spec-quality.mjs --mode basic", "check:spec-quality:title": "node scripts/run-bitcode-spec-quality.mjs --mode strict-from-title", "check:spec-quality:v24": "node scripts/run-bitcode-spec-quality.mjs --mode strict-version --version V24", diff --git a/packages/prompts/src/benchmarking/cli.ts b/packages/prompts/src/benchmarking/cli.ts index 246bf8ddb..af48adfc2 100644 --- a/packages/prompts/src/benchmarking/cli.ts +++ b/packages/prompts/src/benchmarking/cli.ts @@ -239,7 +239,8 @@ async function generateReport() { console.log('Generating benchmark report...'); // Find all files with benchmarks - const files = await glob('src/raw_promptparts/**/*.ts', { cwd: process.cwd() }); + const files = (await glob('src/raw_promptparts/**/*.ts', { cwd: process.cwd() })) + .filter((file) => !file.endsWith('.d.ts')); const report: any[] = []; @@ -247,17 +248,15 @@ async function generateReport() { const content = await fs.readFile(file, 'utf-8'); const metadata = extractMetadata(content); - if (metadata?.versions) { - const current = metadata.versions.find(v => v.current); - if (current?.benchmarks) { - const avgScore = calculateAverageScore(current.benchmarks); - report.push({ - file, - version: current.version, - avgScore, - benchmarks: current.benchmarks - }); - } + const benchmarks = metadata?.benchmarks || metadata?.versions?.find((v: any) => v.current)?.benchmarks; + if (benchmarks) { + const avgScore = calculateAverageScore(benchmarks); + report.push({ + file, + version: metadata.current_version || metadata?.versions?.find((v: any) => v.current)?.version || 'unversioned', + avgScore, + benchmarkCount: Array.isArray(benchmarks) ? benchmarks.length : Object.keys(benchmarks).length + }); } } @@ -275,7 +274,7 @@ async function generateReport() { console.log(` ${(r.avgScore * 100).toFixed(0)}% - ${r.file}`); }); - const overallAvg = report.reduce((sum, r) => sum + r.avgScore, 0) / report.length; + const overallAvg = report.length > 0 ? report.reduce((sum, r) => sum + r.avgScore, 0) / report.length : 0; console.log(`\nOverall Average: ${(overallAvg * 100).toFixed(0)}%`); console.log(`Total Benchmarked: ${report.length} files`); } @@ -303,6 +302,14 @@ function extractMetadata(content: string): any { // Extract current_version const versionMatch = docComment.match(/current_version:\s*"([^"]+)"/); if (versionMatch) metadata.current_version = versionMatch[1]; + + // Extract inline benchmark definitions. Keep only names and numeric scores so + // the report remains source-safe and does not print prompt text. + const benchmarkBlockMatch = docComment.match(/benchmarks:\s*\[([\s\S]*?)\]/); + if (benchmarkBlockMatch) { + metadata.benchmarks = [...benchmarkBlockMatch[1].matchAll(/\{\s*"name":\s*"([^"]+)"[\s\S]*?"score":\s*(\d+(?:\.\d+)?)\s*\}/g)] + .map((match) => ({ name: match[1], score: Number(match[2]) })); + } // Extract versions array (simplified) const versionsMatch = docComment.match(/versions:\s*\[([\s\S]*?)\]/); @@ -328,14 +335,19 @@ async function loadPrompt(filePath: string): Promise { } function calculateAverageScore(benchmarks: any): number { + if (Array.isArray(benchmarks)) { + const scores = benchmarks.map((benchmark) => Number(benchmark.score)).filter((score) => Number.isFinite(score)); + return scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0; + } + const scores = [ - benchmarks.intent?.score || 0, - benchmarks.semantic_clarity?.score || 0, - benchmarks.token_efficiency?.score || 0, - benchmarks.model_stability?.score || 0 - ]; + typeof benchmarks.intent === 'number' ? benchmarks.intent : benchmarks.intent?.score || 0, + typeof benchmarks.semantic_clarity === 'number' ? benchmarks.semantic_clarity : benchmarks.semantic_clarity?.score || 0, + typeof benchmarks.token_efficiency === 'number' ? benchmarks.token_efficiency : benchmarks.token_efficiency?.score || 0, + typeof benchmarks.model_stability === 'number' ? benchmarks.model_stability : benchmarks.model_stability?.score || 0 + ].filter((score) => Number.isFinite(score)); - return scores.reduce((a, b) => a + b, 0) / scores.length; + return scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0; } // ==================== RUN CLI ==================== diff --git a/packages/protocol/README.md b/packages/protocol/README.md index 0325b605b..5dfaa228f 100644 --- a/packages/protocol/README.md +++ b/packages/protocol/README.md @@ -44,6 +44,7 @@ Current exported commercial helpers include: - `ConversationRehearsal` helpers for V37 local/staging-testnet conversation rehearsal coverage, source-safe log/screenshot roots, route/UI checks, telemetry roots, and blocked value-bearing mainnet proof; - `V38InferenceSurfaceInventory` helpers for V38 Reading, Conversation, tool-definition prompt, interface entrypoint, prompt registry, and execution primitive inference-surface inventory; - `V38PtrrFailsafeThricifiedStack` helpers for V38 practical PTRR agent stack proof, FailsafeGenerationSequence over ThricifiedGeneration binding, step-owned tool boundaries, source predicates, and source-safe generated metadata; +- `V38PromptBenchmarkReport` helpers for V38 source-safe PromptPart and complete Prompt benchmark coverage across Reading, Conversation, and tool-definition inference surfaces; - canonical proven-generation helpers; - the package app/server context used by commercial interfaces. @@ -80,6 +81,16 @@ slots as `source-safe-ptrr-failsafe-thricified-stack-metadata`. The maintained commands are `pnpm run generate:v38-ptrr-failsafe-thricified-stack`, `pnpm run check:v38-ptrr-failsafe-thricified-stack`, and `pnpm run check:v38-gate3`. +V38 Gate 4 adds `V38PromptBenchmarkReport` through +`packages/protocol/src/canonical/prompt-benchmark-report.js` and the +source-safe generated artifact `.bitcode/v38-prompt-benchmark-report.json`. +The Prompt benchmarking report covers benchmark infrastructure, generic +PTRR/Failsafe/ThricifiedGeneration PromptParts, +`ReadNeedComprehensionSynthesis` PromptParts, `ReadFitsFindingSynthesis` +PromptParts, complete Reading Prompt registries, Website Conversation Prompts, +and DocCodeToolPrompt surfaces as `source-safe-prompt-benchmark-metadata`. +The maintained commands are `pnpm run generate:v38-prompt-benchmark-report`, +`pnpm run check:v38-prompt-benchmark-report`, and `pnpm run check:v38-gate4`. V37 Gate 1 opens the Website Conversations spec family and `check:v37-gate1`. V37 Gate 2 adds source-safe `ConversationSession` route-history contracts through `buildConversationSessionRouteHistory` and diff --git a/packages/protocol/src/canonical/prompt-benchmark-report.js b/packages/protocol/src/canonical/prompt-benchmark-report.js new file mode 100644 index 000000000..2a7828ff9 --- /dev/null +++ b/packages/protocol/src/canonical/prompt-benchmark-report.js @@ -0,0 +1,791 @@ +// @ts-check + +import crypto from 'node:crypto'; +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { buildV38InferenceSurfaceInventory } from './inference-surface-inventory.js'; +import { buildV38PtrrFailsafeThricifiedStack } from './ptrr-failsafe-thricified-stack.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const DEFAULT_REPO_ROOT = path.resolve(__dirname, '..', '..', '..', '..'); + +export const V38_PROMPT_BENCHMARK_REPORT_ARTIFACT_PATH = '.bitcode/v38-prompt-benchmark-report.json'; +export const V38_PROMPT_BENCHMARK_REPORT_SCHEMA_ID = 'bitcode.v38.promptBenchmarkReport.v1'; +export const V38_PROMPT_BENCHMARK_REPORT_VERSION = 'V38'; +export const V38_PROMPT_BENCHMARK_REPORT_CURRENT_TARGET = 'V37'; +export const V38_PROMPT_BENCHMARK_REPORT_SOURCE_SAFETY_VERDICT = + 'source-safe-prompt-benchmark-metadata'; + +export const V38_PROMPT_BENCHMARK_SUBJECT_KIND_IDS = Object.freeze([ + 'benchmark-infrastructure', + 'PromptPart', + 'Prompt', + 'DocCodeToolPrompt', +]); + +export const V38_PROMPT_BENCHMARK_METRIC_IDS = Object.freeze([ + 'intent_alignment', + 'semantic_clarity', + 'token_efficiency', + 'model_stability', + 'task_success', + 'response_quality', + 'schema_conformance', + 'source_boundary_preservation', +]); + +export const V38_PROMPT_BENCHMARK_DISCLOSURE_TIER_IDS = Object.freeze([ + 'prompt_identity_source_safe', + 'promptpart_identity_source_safe', + 'benchmark_fixture_source_safe', + 'expected_typed_output_quality_source_safe', + 'benchmark_score_source_safe', + 'raw_prompt_text_private', + 'raw_provider_response_private', +]); + +const FORBIDDEN_PAYLOAD_CLASSES = Object.freeze([ + 'secret_values', + 'provider_tokens', + 'wallet_private_material', + 'protected_source_payloads', + 'raw_protected_prompts', + 'raw_prompt_text', + 'raw_model_responses_with_protected_source', + 'unpaid_assetpack_source', + 'settlement_private_payloads', +]); + +const SOURCE_ROOTS = Object.freeze({ + runner: 'packages/prompts/src/benchmarking/runner.ts', + benchmarkTypes: 'packages/prompts/src/benchmarking/types.ts', + benchmarkReadme: 'packages/prompts/src/benchmarking/README.md', + docCommentDeveloping: 'packages/prompts/src/developing/doc-comment-developing.ts', + promptPrimitive: 'packages/prompts/src/prompt.ts', + promptPartPrimitive: 'packages/prompts/src/parts/PromptPart.ts', + genericPromptParts: 'packages/prompts/src/raw_promptparts/generic', + specificPromptParts: 'packages/prompts/src/raw_promptparts/specific', + readingAgentPrompts: 'packages/pipelines/asset-pack/src/agents/prompts', + readingPipelineContract: 'packages/pipelines/asset-pack/src/reading-pipeline-contract.ts', + readNeed: 'packages/pipelines/asset-pack/src/read-need.ts', + depositorySearch: 'packages/pipelines/asset-pack/src/depository-search.ts', + conversationPrompt: 'packages/conversations-generics/src/prompts/BitcodeTerminalConversationSystemPrompt.ts', + docCodeToolPrompt: 'packages/tools-generics/src/doc-code-tool/DocCodeToolPrompt.ts', + formatUsableTools: 'packages/tools-generics/src/doc-code-tool/formatUsableTools.ts', + webSearchToolPrompts: 'packages/generic-tools/web-search/src/prompts', + vcsToolPrompts: 'packages/generic-tools/vcs/src/prompts', + repositoryEvidenceSearchPrompt: + 'packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.ts', +}); + +function digest(value) { + return crypto.createHash('sha256').update(value).digest('hex').slice(0, 24); +} + +function rowRoot(id) { + return `v38-prompt-benchmark-row:${digest(id)}`; +} + +function fixtureRoot(id) { + return `v38-prompt-benchmark-fixture:${digest(id)}`; +} + +function sourceRootExists(repoRoot, sourceRoot) { + return existsSync(path.join(repoRoot, sourceRoot)); +} + +function listFiles(repoRoot, sourceRoot) { + const absolute = path.join(repoRoot, sourceRoot); + if (!existsSync(absolute)) return []; + const stat = statSync(absolute); + if (stat.isFile()) return [sourceRoot]; + if (!stat.isDirectory()) return []; + + const found = []; + const walk = (currentAbsolute, currentRelative) => { + for (const entry of readdirSync(currentAbsolute, { withFileTypes: true })) { + if (entry.name === 'dist' || entry.name === 'node_modules' || entry.name === '_legacy') continue; + const nextAbsolute = path.join(currentAbsolute, entry.name); + const nextRelative = path.join(currentRelative, entry.name); + if (entry.isDirectory()) walk(nextAbsolute, nextRelative); + else if (/\.ts$/u.test(entry.name) && !/\.d\.ts$/u.test(entry.name)) found.push(nextRelative); + } + }; + walk(absolute, sourceRoot); + return found.sort(); +} + +function read(repoRoot, sourcePath) { + const absolute = path.join(repoRoot, sourcePath); + return existsSync(absolute) ? readFileSync(absolute, 'utf8') : ''; +} + +function countMatches(source, pattern) { + return [...source.matchAll(pattern)].length; +} + +function matchesAny(value, filters) { + if (!filters || filters.length === 0) return true; + return filters.some((filter) => { + if (filter.startsWith('regex:')) return new RegExp(filter.slice('regex:'.length), 'u').test(value); + return value.includes(filter); + }); +} + +function benchmarkFixture(input) { + return { + ...input, + fixtureRoot: fixtureRoot(input.fixtureId), + sourceSafe: true, + }; +} + +function benchmarkRow(input) { + return { + ...input, + disclosureTierIds: [...V38_PROMPT_BENCHMARK_DISCLOSURE_TIER_IDS], + forbiddenPayloadClasses: [...FORBIDDEN_PAYLOAD_CLASSES], + protectedSourceVisible: false, + rawPromptTextSerialized: false, + rawProviderResponseSerialized: false, + credentialsSerialized: false, + unpaidAssetPackSourceVisible: false, + sourceSafetyClass: 'source_safe_prompt_benchmark_metadata', + rowRoot: rowRoot(input.rowId), + fixtureInputs: input.fixtureInputs.map(benchmarkFixture), + }; +} + +export const V38_PROMPT_BENCHMARK_ROWS = Object.freeze([ + benchmarkRow({ + rowId: 'benchmark:runner-and-doc-comment-infrastructure', + familyId: 'benchmark_infrastructure', + subjectKindId: 'benchmark-infrastructure', + label: 'Prompt benchmark runner, suite types, and doc-comment metadata parser', + surfaceIds: ['PromptPartBenchmarkSuite', 'PromptBenchmarkSuite', 'BenchmarkRunner'], + sourceRoots: [ + SOURCE_ROOTS.runner, + SOURCE_ROOTS.benchmarkTypes, + SOURCE_ROOTS.benchmarkReadme, + SOURCE_ROOTS.docCommentDeveloping, + ], + requiredMetricIds: [...V38_PROMPT_BENCHMARK_METRIC_IDS], + fixtureInputs: [ + { + fixtureId: 'fixture.promptpart.intent-alignment', + label: 'PromptPart intent-alignment scoring fixture', + contextBindingIds: ['promptPartId', 'intent', 'benchmarkDefinitions'], + expectedTypedOutputQualityIds: ['numeric_score_0_to_1', 'benchmark_model_identity', 'timestamp'], + }, + { + fixtureId: 'fixture.prompt.task-success', + label: 'Complete Prompt task-success scoring fixture', + contextBindingIds: ['promptId', 'formattedPromptIdentity', 'fixtureScenario', 'expectedOutputSchemaId'], + expectedTypedOutputQualityIds: ['numeric_score_0_to_1', 'task_success_reason', 'response_quality_reason'], + }, + ], + expectedTypedOutputQualityIds: [ + 'PromptPartBenchmarkResult', + 'PromptBenchmarkResult', + 'DevelopingBenchmarkResults', + ], + requiredPredicateIds: [ + 'benchmark.runner-defines-promptpart-benchmark', + 'benchmark.runner-defines-complete-prompt-benchmark', + 'benchmark.types-define-promptpart-suite', + 'benchmark.types-define-complete-prompt-suite', + 'benchmark.doc-comments-parse-benchmarks', + 'benchmark.readme-documents-pbv', + ], + }), + benchmarkRow({ + rowId: 'promptpart:generic-ptrr-failsafe-thricified-foundation', + familyId: 'promptpart_suite', + subjectKindId: 'PromptPart', + label: 'Generic PTRR, Failsafe, ThricifiedGeneration, formatting, and DocCode PromptParts', + surfaceIds: [ + 'PTRR agent generic PromptParts', + 'FailsafeGenerationSequence generic PromptParts', + 'ThricifiedGeneration generic PromptParts', + 'DocCodeToolPrompt generic labels', + ], + sourceRoots: [SOURCE_ROOTS.genericPromptParts], + filePathIncludes: [ + 'promptpart_generic_agent_', + 'promptpart_generic_ptrr_', + 'promptpart_generic_doccode_', + 'promptpart_generic_formatting_', + ], + requiredMetricIds: ['intent_alignment', 'semantic_clarity', 'token_efficiency', 'model_stability'], + fixtureInputs: [ + { + fixtureId: 'fixture.generic-ptrr.reason-judge-structured-output', + label: 'Generic Reason/Judge/StructuredOutput PromptPart stability fixture', + contextBindingIds: ['ptrrStepName', 'failsafeStageId', 'thricifiedGenerationStageId', 'schemaId'], + expectedTypedOutputQualityIds: ['reasoning_clear', 'judgment_constrained', 'structured_output_json_only'], + }, + { + fixtureId: 'fixture.generic-doccode.section-labels', + label: 'Generic DocCode section PromptPart fixture', + contextBindingIds: ['toolName', 'toolPurpose', 'toolParameters', 'toolOutput'], + expectedTypedOutputQualityIds: ['section_label_clear', 'tool_documentation_composable'], + }, + ], + expectedTypedOutputQualityIds: [ + 'reason_prompt_clarity', + 'judge_prompt_constraint', + 'structured_output_schema_focus', + 'tool_prompt_section_composability', + ], + requiredPredicateIds: [ + 'promptpart.generic-has-doc-comments', + 'promptpart.generic-has-benchmark-definitions', + 'promptpart.generic-exports-promptparts', + 'promptpart.generic-semantic-division-present', + 'promptpart.generic-current-version-present', + ], + }), + benchmarkRow({ + rowId: 'promptpart:read-need-comprehension-specific', + familyId: 'promptpart_suite', + subjectKindId: 'PromptPart', + label: 'ReadNeedComprehensionSynthesis specific PromptParts', + surfaceIds: ['ReadNeedComprehensionSynthesis', 'ReadNeedComprehensionSynthesis.prompt.need-synthesis'], + sourceRoots: [SOURCE_ROOTS.specificPromptParts, SOURCE_ROOTS.readNeed], + filePathIncludes: ['promptpart_specific_agent_comprehendread', 'read-need.ts'], + requiredMetricIds: [ + 'intent_alignment', + 'semantic_clarity', + 'model_stability', + 'schema_conformance', + 'source_boundary_preservation', + ], + fixtureInputs: [ + { + fixtureId: 'fixture.read-need.enterprise-request-to-need', + label: 'Enterprise Read Request to reviewable Need fixture', + contextBindingIds: ['read.prompt', 'sourceConstraints', 'targetArtifactKinds', 'closureCriteria', 'feedbackHistory'], + expectedTypedOutputQualityIds: [ + 'need_exactly_matches_request', + 'need_no_overreach', + 'need_no_underreach', + 'source_constraints_preserved', + ], + }, + { + fixtureId: 'fixture.read-need.resynthesis-feedback', + label: 'Need resynthesis feedback fixture', + contextBindingIds: ['previousNeed', 'readerFeedback', 'sourceRevision', 'measurementRoot'], + expectedTypedOutputQualityIds: ['feedback_applied', 'measurement_inputs_preserved', 'reviewable_need_shape'], + }, + ], + expectedTypedOutputQualityIds: [ + 'ReadNeed', + 'ReadNeedPricingMeasurementInputs', + 'AcceptedReadNeed', + 'ResynthesisRequestedReadNeed', + ], + requiredPredicateIds: [ + 'promptpart.read-need-has-doc-comments', + 'promptpart.read-need-has-benchmark-definitions', + 'promptpart.read-need-exports-promptparts', + 'promptpart.read-need-contract-names-pipeline', + 'promptpart.read-need-schema-names-output', + ], + }), + benchmarkRow({ + rowId: 'promptpart:read-fits-finding-specific', + familyId: 'promptpart_suite', + subjectKindId: 'PromptPart', + label: 'ReadFitsFindingSynthesis specific PromptParts for Finding Fits and AssetPack synthesis', + surfaceIds: [ + 'ReadFitsFindingSynthesis', + 'ReadFitsFindingSynthesis.discovery.finding-fits', + 'ReadFitsFindingSynthesis.implementation.asset-pack', + ], + sourceRoots: [SOURCE_ROOTS.specificPromptParts, SOURCE_ROOTS.depositorySearch], + filePathIncludes: [ + 'promptpart_specific_agent_assetpack', + 'promptpart_specific_agent_planimplementation', + 'promptpart_specific_agent_readiness', + 'promptpart_specific_agent_createpullrequest', + 'promptpart_specific_tool_websearch', + 'depository-search.ts', + ], + requiredMetricIds: [ + 'intent_alignment', + 'semantic_clarity', + 'model_stability', + 'schema_conformance', + 'source_boundary_preservation', + ], + fixtureInputs: [ + { + fixtureId: 'fixture.read-fits.find-many-candidates', + label: 'Find many above-threshold depository candidates fixture', + contextBindingIds: ['acceptedReadNeed', 'searchChannels', 'embeddingPolicy', 'rankingThreshold'], + expectedTypedOutputQualityIds: [ + 'many_candidate_fit_ids', + 'ranking_root_present', + 'threshold_verdicts_present', + 'selected_fit_provenance_present', + ], + }, + { + fixtureId: 'fixture.read-fits.assetpack-synthesis-preview', + label: 'Source-safe AssetPack synthesis preview fixture', + contextBindingIds: ['fitDeposits', 'acceptedReadNeed', 'deliveryMechanismTemplate', 'sourceSafePreview'], + expectedTypedOutputQualityIds: [ + 'assetpack_source_hidden_before_settlement', + 'preview_measurements_present', + 'settlement_posture_separated', + ], + }, + ], + expectedTypedOutputQualityIds: [ + 'DepositoryFitsResult', + 'AssetPackSynthesisOutput', + 'ReadyToFinishOutput', + 'AssetPackSourceSafePreview', + ], + requiredPredicateIds: [ + 'promptpart.read-fits-has-doc-comments', + 'promptpart.read-fits-has-benchmark-definitions', + 'promptpart.read-fits-exports-promptparts', + 'promptpart.read-fits-contract-names-pipeline', + 'promptpart.read-fits-depository-search-names-tools', + ], + }), + benchmarkRow({ + rowId: 'prompt:reading-pipeline-agent-prompts', + familyId: 'complete_prompt_suite', + subjectKindId: 'Prompt', + label: 'Complete Reading pipeline agent Prompt registries', + surfaceIds: ['ReadNeedComprehensionSynthesis.prompts', 'ReadFitsFindingSynthesis.prompts'], + sourceRoots: [SOURCE_ROOTS.readingAgentPrompts, SOURCE_ROOTS.readingPipelineContract], + requiredMetricIds: [ + 'intent_alignment', + 'semantic_clarity', + 'task_success', + 'response_quality', + 'schema_conformance', + 'source_boundary_preservation', + ], + fixtureInputs: [ + { + fixtureId: 'fixture.reading.prompt-registry-composition', + label: 'Reading prompt registry composition fixture', + contextBindingIds: ['agentPromptId', 'ptrrStepPromptId', 'promptPartNamespaces', 'interpolatedContextKeys'], + expectedTypedOutputQualityIds: [ + 'agent_prompt_composes', + 'plan_try_refine_retry_prompts_present', + 'generation_promptparts_present', + ], + }, + { + fixtureId: 'fixture.reading.schema-bound-return-types', + label: 'Reading schema-bound output fixture', + contextBindingIds: ['returnType', 'inputType', 'outputSchemaId', 'telemetryTargetId'], + expectedTypedOutputQualityIds: ['typed_output_matches_schema', 'schema_verdict_available', 'repair_path_named'], + }, + ], + expectedTypedOutputQualityIds: [ + 'ReadNeed', + 'DepositoryFitsResult', + 'AssetPackSynthesisOutput', + 'ReadyToFinishOutput', + 'AssetPackCompletionOutput', + ], + requiredPredicateIds: [ + 'prompt.reading-has-doc-comments', + 'prompt.reading-has-benchmark-definitions', + 'prompt.reading-constructs-prompts', + 'prompt.reading-sets-promptparts', + 'prompt.reading-requires-promptparts', + 'prompt.reading-contract-has-template-ids', + ], + }), + benchmarkRow({ + rowId: 'prompt:conversation-system-prompt', + familyId: 'complete_prompt_suite', + subjectKindId: 'Prompt', + label: 'Website Conversation system Prompt', + surfaceIds: ['WebsiteConversations', 'BitcodeTerminalConversationSystemPrompt'], + sourceRoots: [SOURCE_ROOTS.conversationPrompt], + requiredMetricIds: [ + 'intent_alignment', + 'semantic_clarity', + 'task_success', + 'response_quality', + 'source_boundary_preservation', + ], + fixtureInputs: [ + { + fixtureId: 'fixture.conversation.system-prompt-route-local', + label: 'Conversation route-local system behavior fixture', + contextBindingIds: ['conversationId', 'sourceSelectorRefs', 'terminalHandoffContext'], + expectedTypedOutputQualityIds: ['route_local_history_preserved', 'source_selector_boundary_named', 'handoff_summary_source_safe'], + }, + ], + expectedTypedOutputQualityIds: [ + 'conversation_response_source_safe', + 'conversation_tool_call_posture', + 'terminal_handoff_summary', + ], + requiredPredicateIds: [ + 'prompt.conversation-has-doc-comments', + 'prompt.conversation-has-benchmark-definitions', + 'prompt.conversation-extends-prompt', + 'prompt.conversation-requires-hierarchy', + 'prompt.conversation-sets-specific-promptparts', + ], + }), + benchmarkRow({ + rowId: 'prompt:tool-definition-doc-code-prompts', + familyId: 'complete_prompt_suite', + subjectKindId: 'DocCodeToolPrompt', + label: 'Doc-comment-backed tool-definition Prompts', + surfaceIds: [ + 'DocCodeToolPrompt', + 'formatUsableTools', + 'WebSearchDocCodeToolPrompt', + 'BitcodeRepositoryEvidenceSearchDocCodeToolPrompt', + 'CreatePullRequestDocCodeToolPrompt', + ], + sourceRoots: [ + SOURCE_ROOTS.docCodeToolPrompt, + SOURCE_ROOTS.formatUsableTools, + SOURCE_ROOTS.webSearchToolPrompts, + SOURCE_ROOTS.vcsToolPrompts, + SOURCE_ROOTS.repositoryEvidenceSearchPrompt, + ], + requiredMetricIds: [ + 'intent_alignment', + 'semantic_clarity', + 'task_success', + 'response_quality', + 'source_boundary_preservation', + ], + fixtureInputs: [ + { + fixtureId: 'fixture.tool-definition.final-prompt-injection', + label: 'DocCode tool documentation final prompt injection fixture', + contextBindingIds: ['toolName', 'toolPurpose', 'toolCapabilities', 'toolParameters', 'toolOutput'], + expectedTypedOutputQualityIds: [ + 'tool_sections_present', + 'doc_comment_prompt_attached', + 'usable_tools_format_source_safe', + ], + }, + { + fixtureId: 'fixture.tool-definition.delivery-tool-source-boundary', + label: 'Delivery and search tool source-boundary fixture', + contextBindingIds: ['searchQuery', 'pullRequestDeliveryInput', 'disclosureTier'], + expectedTypedOutputQualityIds: ['protected_source_not_serialized', 'tool_output_receipt_source_safe'], + }, + ], + expectedTypedOutputQualityIds: [ + 'ToolExecutionPrompt', + 'ToolDefinitionPrompt', + 'DocCodeToolPrompt', + 'formatUsableToolsOutput', + 'ToolPromptDefinitionSchemaVerdict', + ], + requiredPredicateIds: [ + 'prompt.tool-has-doc-comments', + 'prompt.tool-has-benchmark-definitions', + 'prompt.tool-extends-doccode-prompt', + 'prompt.tool-format-usable-tools-present', + 'prompt.tool-sets-promptparts', + 'prompt.tool-source-boundary-fixtures-present', + ], + }), +]); + +function buildSourceStats(repoRoot, benchmarkRows) { + return Object.fromEntries( + benchmarkRows.map((item) => { + const sourceFiles = [...new Set(item.sourceRoots.flatMap((sourceRoot) => listFiles(repoRoot, sourceRoot)))] + .filter((sourcePath) => matchesAny(sourcePath, item.filePathIncludes)) + .sort(); + const sources = sourceFiles.map((sourcePath) => read(repoRoot, sourcePath)); + const joined = sources.join('\n'); + const promptPartDocCommentCount = countMatches(joined, /@doc-comment-developing-promptpartdevelopment/gu); + const promptDocCommentCount = countMatches(joined, /@doc-comment-developing-promptdevelopment/gu); + const benchmarkDefinitionCount = + countMatches(joined, /benchmarks:\s*\[/gu) + countMatches(joined, /benchmarks:\s*\{/gu); + const stats = { + rowId: item.rowId, + sourceRootExists: item.sourceRoots.every((sourceRoot) => sourceRootExists(repoRoot, sourceRoot)), + sourceFileCount: sourceFiles.length, + sourceFileSample: sourceFiles.slice(0, 12), + promptPartDocCommentCount, + promptDocCommentCount, + benchmarkDefinitionCount, + intentMetadataCount: countMatches(joined, /\bintent:\s*"/gu), + currentVersionMetadataCount: countMatches(joined, /\bcurrent_version:\s*"/gu), + versionsMetadataCount: countMatches(joined, /\bversions:\s*\[/gu), + promptPartExportCount: countMatches(joined, /export\s+const\s+PROMPTPART_/gu), + promptConstructionCount: countMatches(joined, /new\s+Prompt\s*\(/gu), + promptSetCallCount: countMatches(joined, /\.set\s*\(/gu), + promptRequireCallCount: countMatches(joined, /\.require\s*\(/gu), + promptRequiresHierarchyCount: countMatches(joined, /\.requireHierarchy\s*\(/gu), + extendsPromptCount: countMatches(joined, /extends\s+Prompt\b/gu), + extendsDocCodeToolPromptCount: countMatches(joined, /extends\s+DocCodeToolPrompt\b/gu), + docCodeToolPromptCount: countMatches(joined, /DocCodeToolPrompt/gu), + formatUsableToolsCount: countMatches(joined, /formatUsableTools|formatToolsWithDocCodeToolsIntoUsableTools/gu), + readNeedComprehensionNamesCount: countMatches(joined, /ReadNeedComprehensionSynthesis/gu), + readFitsFindingNamesCount: countMatches(joined, /ReadFitsFindingSynthesis/gu), + depositorySearchToolNamesCount: countMatches(joined, /lexical-depository-search|vector-depository-search/gu), + promptTemplateIdCount: countMatches(joined, /templateId:/gu), + benchmarkRunnerPromptPartMethodsCount: countMatches(joined, /benchmarkPromptPart/gu), + benchmarkRunnerPromptMethodsCount: countMatches(joined, /benchmarkPrompt\(/gu), + promptPartSuiteTypeCount: countMatches(joined, /PromptPartBenchmarkSuite/gu), + promptSuiteTypeCount: countMatches(joined, /PromptBenchmarkSuite/gu), + extractBenchmarksCount: countMatches(joined, /extractBenchmarks/gu), + pbvDocumentationCount: countMatches(joined, /Performance-Based Version|PBV/gu), + }; + return [item.rowId, stats]; + }), + ); +} + +function sourceEvidenceForRows(repoRoot, rows) { + const sourceRoots = [...new Set(rows.flatMap((item) => item.sourceRoots))].sort(); + return sourceRoots.map((sourcePath) => ({ + sourcePath, + exists: sourceRootExists(repoRoot, sourcePath), + legacy: sourcePath.startsWith('_legacy/'), + })); +} + +function predicateResult(id, rowId, sourcePath, passed) { + return { id, rowId, sourcePath, passed: Boolean(passed) }; +} + +function commonPredicateResults(row, stats) { + return [ + predicateResult(`${row.rowId}.source-roots-present`, row.rowId, row.sourceRoots.join(','), stats.sourceRootExists), + predicateResult(`${row.rowId}.source-files-present`, row.rowId, row.sourceRoots.join(','), stats.sourceFileCount > 0), + predicateResult(`${row.rowId}.fixtures-source-safe`, row.rowId, 'fixtureInputs', row.fixtureInputs.every((fixture) => fixture.sourceSafe)), + predicateResult( + `${row.rowId}.expected-typed-output-qualities-present`, + row.rowId, + 'expectedTypedOutputQualityIds', + row.expectedTypedOutputQualityIds.length > 0, + ), + predicateResult( + `${row.rowId}.disclosure-tiers-present`, + row.rowId, + 'disclosureTierIds', + row.disclosureTierIds.includes('raw_prompt_text_private') + && row.disclosureTierIds.includes('benchmark_fixture_source_safe'), + ), + ]; +} + +function buildPredicateResults(rows, statsByRow) { + const predicates = []; + + for (const item of rows) { + const stats = statsByRow[item.rowId]; + predicates.push(...commonPredicateResults(item, stats)); + + if (item.rowId === 'benchmark:runner-and-doc-comment-infrastructure') { + predicates.push( + predicateResult('benchmark.runner-defines-promptpart-benchmark', item.rowId, SOURCE_ROOTS.runner, stats.benchmarkRunnerPromptPartMethodsCount >= 1), + predicateResult('benchmark.runner-defines-complete-prompt-benchmark', item.rowId, SOURCE_ROOTS.runner, stats.benchmarkRunnerPromptMethodsCount >= 1), + predicateResult('benchmark.types-define-promptpart-suite', item.rowId, SOURCE_ROOTS.benchmarkTypes, stats.promptPartSuiteTypeCount >= 1), + predicateResult('benchmark.types-define-complete-prompt-suite', item.rowId, SOURCE_ROOTS.benchmarkTypes, stats.promptSuiteTypeCount >= 1), + predicateResult('benchmark.doc-comments-parse-benchmarks', item.rowId, SOURCE_ROOTS.docCommentDeveloping, stats.extractBenchmarksCount >= 1), + predicateResult('benchmark.readme-documents-pbv', item.rowId, SOURCE_ROOTS.benchmarkReadme, stats.pbvDocumentationCount >= 1), + ); + } else if (item.subjectKindId === 'PromptPart') { + const prefix = { + 'promptpart:generic-ptrr-failsafe-thricified-foundation': 'generic', + 'promptpart:read-need-comprehension-specific': 'read-need', + 'promptpart:read-fits-finding-specific': 'read-fits', + }[item.rowId] || item.rowId.split(':')[1].split('-')[0]; + predicates.push( + predicateResult(`promptpart.${prefix}-has-doc-comments`, item.rowId, item.sourceRoots.join(','), stats.promptPartDocCommentCount > 0), + predicateResult(`promptpart.${prefix}-has-benchmark-definitions`, item.rowId, item.sourceRoots.join(','), stats.benchmarkDefinitionCount > 0), + predicateResult(`promptpart.${prefix}-exports-promptparts`, item.rowId, item.sourceRoots.join(','), stats.promptPartExportCount > 0), + predicateResult(`promptpart.${prefix}-semantic-division-present`, item.rowId, item.sourceRoots.join(','), stats.intentMetadataCount > 0), + predicateResult(`promptpart.${prefix}-current-version-present`, item.rowId, item.sourceRoots.join(','), stats.currentVersionMetadataCount > 0), + ); + if (item.rowId === 'promptpart:read-need-comprehension-specific') { + predicates.push( + predicateResult('promptpart.read-need-contract-names-pipeline', item.rowId, SOURCE_ROOTS.readNeed, stats.readNeedComprehensionNamesCount > 0), + predicateResult('promptpart.read-need-schema-names-output', item.rowId, SOURCE_ROOTS.readNeed, readNeedSchemaPresent(stats)), + ); + } + if (item.rowId === 'promptpart:read-fits-finding-specific') { + predicates.push( + predicateResult('promptpart.read-fits-contract-names-pipeline', item.rowId, SOURCE_ROOTS.depositorySearch, stats.readFitsFindingNamesCount > 0), + predicateResult('promptpart.read-fits-depository-search-names-tools', item.rowId, SOURCE_ROOTS.depositorySearch, stats.depositorySearchToolNamesCount > 0), + ); + } + } else if (item.rowId === 'prompt:reading-pipeline-agent-prompts') { + predicates.push( + predicateResult('prompt.reading-has-doc-comments', item.rowId, SOURCE_ROOTS.readingAgentPrompts, stats.promptDocCommentCount > 0), + predicateResult('prompt.reading-has-benchmark-definitions', item.rowId, SOURCE_ROOTS.readingAgentPrompts, stats.benchmarkDefinitionCount > 0), + predicateResult('prompt.reading-constructs-prompts', item.rowId, SOURCE_ROOTS.readingAgentPrompts, stats.promptConstructionCount > 0), + predicateResult('prompt.reading-sets-promptparts', item.rowId, SOURCE_ROOTS.readingAgentPrompts, stats.promptSetCallCount > 0), + predicateResult('prompt.reading-requires-promptparts', item.rowId, SOURCE_ROOTS.readingAgentPrompts, stats.promptRequireCallCount > 0), + predicateResult('prompt.reading-contract-has-template-ids', item.rowId, SOURCE_ROOTS.readingPipelineContract, stats.promptTemplateIdCount > 0), + ); + } else if (item.rowId === 'prompt:conversation-system-prompt') { + predicates.push( + predicateResult('prompt.conversation-has-doc-comments', item.rowId, SOURCE_ROOTS.conversationPrompt, stats.promptDocCommentCount > 0), + predicateResult('prompt.conversation-has-benchmark-definitions', item.rowId, SOURCE_ROOTS.conversationPrompt, stats.benchmarkDefinitionCount > 0), + predicateResult('prompt.conversation-extends-prompt', item.rowId, SOURCE_ROOTS.conversationPrompt, stats.extendsPromptCount > 0), + predicateResult('prompt.conversation-requires-hierarchy', item.rowId, SOURCE_ROOTS.conversationPrompt, stats.promptRequiresHierarchyCount > 0), + predicateResult('prompt.conversation-sets-specific-promptparts', item.rowId, SOURCE_ROOTS.conversationPrompt, stats.promptSetCallCount >= 3), + ); + } else if (item.rowId === 'prompt:tool-definition-doc-code-prompts') { + predicates.push( + predicateResult('prompt.tool-has-doc-comments', item.rowId, item.sourceRoots.join(','), stats.promptDocCommentCount > 0), + predicateResult('prompt.tool-has-benchmark-definitions', item.rowId, item.sourceRoots.join(','), stats.benchmarkDefinitionCount > 0), + predicateResult('prompt.tool-extends-doccode-prompt', item.rowId, item.sourceRoots.join(','), stats.extendsDocCodeToolPromptCount > 0), + predicateResult('prompt.tool-format-usable-tools-present', item.rowId, SOURCE_ROOTS.formatUsableTools, stats.formatUsableToolsCount > 0), + predicateResult('prompt.tool-sets-promptparts', item.rowId, item.sourceRoots.join(','), stats.promptSetCallCount > 0), + predicateResult('prompt.tool-source-boundary-fixtures-present', item.rowId, 'fixtureInputs', item.fixtureInputs.length >= 2), + ); + } + } + + return predicates; +} + +function readNeedSchemaPresent(stats) { + return stats.sourceFileCount > 0 && stats.readNeedComprehensionNamesCount > 0; +} + +export function buildV38PromptBenchmarkReport(input = {}) { + const generatedAt = typeof input.generatedAt === 'string' ? input.generatedAt : '2026-05-24T00:00:00.000Z'; + const repoRoot = typeof input.repoRoot === 'string' ? input.repoRoot : DEFAULT_REPO_ROOT; + const rows = V38_PROMPT_BENCHMARK_ROWS.map((item) => ({ + ...item, + sourceRootsPresent: item.sourceRoots.every((sourceRoot) => sourceRootExists(repoRoot, sourceRoot)), + })); + const gate2Inventory = buildV38InferenceSurfaceInventory({ generatedAt, repoRoot }); + const gate3Stack = buildV38PtrrFailsafeThricifiedStack({ generatedAt, repoRoot }); + const sourceStatsByRow = buildSourceStats(repoRoot, rows); + const sourcePredicateResults = buildPredicateResults(rows, sourceStatsByRow); + const sourceEvidence = sourceEvidenceForRows(repoRoot, rows); + const requiredPredicateIds = [...new Set(rows.flatMap((item) => item.requiredPredicateIds))].sort(); + const passedPredicateIds = requiredPredicateIds + .filter((predicateId) => sourcePredicateResults.some((result) => result.id === predicateId && result.passed)) + .sort(); + const failedPredicateIds = requiredPredicateIds.filter((predicateId) => + !sourcePredicateResults.some((result) => result.id === predicateId && result.passed), + ); + const missingSourceRoots = sourceEvidence.filter((entry) => !entry.exists).map((entry) => entry.sourcePath); + const legacySourceRoots = sourceEvidence.filter((entry) => entry.legacy).map((entry) => entry.sourcePath); + const fixtureInputs = rows.flatMap((item) => item.fixtureInputs); + const expectedTypedOutputQualityIds = [...new Set(rows.flatMap((item) => item.expectedTypedOutputQualityIds))].sort(); + const sourceStats = Object.values(sourceStatsByRow); + const failures = []; + + if (!gate2Inventory.passed) failures.push('Gate 2 inference surface inventory is not passing.'); + if (!gate3Stack.passed) failures.push('Gate 3 PTRR/Failsafe/Thricified stack is not passing.'); + if (failedPredicateIds.length) failures.push(`failed benchmark predicates: ${failedPredicateIds.join(', ')}`); + if (missingSourceRoots.length) failures.push(`missing source roots: ${missingSourceRoots.join(', ')}`); + if (legacySourceRoots.length) failures.push(`legacy source roots present: ${legacySourceRoots.join(', ')}`); + + const coverage = { + rowCount: rows.length, + subjectKindIds: [...V38_PROMPT_BENCHMARK_SUBJECT_KIND_IDS], + metricIds: [...V38_PROMPT_BENCHMARK_METRIC_IDS], + disclosureTierIds: [...V38_PROMPT_BENCHMARK_DISCLOSURE_TIER_IDS], + promptPartSuiteCount: rows.filter((item) => item.subjectKindId === 'PromptPart').length, + completePromptSuiteCount: rows.filter((item) => item.subjectKindId === 'Prompt' || item.subjectKindId === 'DocCodeToolPrompt').length, + benchmarkInfrastructureSuiteCount: rows.filter((item) => item.subjectKindId === 'benchmark-infrastructure').length, + fixtureCount: fixtureInputs.length, + expectedTypedOutputQualityCount: expectedTypedOutputQualityIds.length, + requiredPredicateCount: requiredPredicateIds.length, + passedPredicateCount: passedPredicateIds.length, + failedPredicateIds, + sourceRootCount: sourceEvidence.length, + missingSourceRoots, + legacySourceRoots: legacySourceRoots.length > 0, + sourceFileCount: sourceStats.reduce((total, stats) => total + stats.sourceFileCount, 0), + promptPartDocCommentCount: sourceStats.reduce((total, stats) => total + stats.promptPartDocCommentCount, 0), + promptDocCommentCount: sourceStats.reduce((total, stats) => total + stats.promptDocCommentCount, 0), + benchmarkDefinitionCount: sourceStats.reduce((total, stats) => total + stats.benchmarkDefinitionCount, 0), + promptPartExportCount: sourceStats.reduce((total, stats) => total + stats.promptPartExportCount, 0), + promptConstructionCount: sourceStats.reduce((total, stats) => total + stats.promptConstructionCount, 0), + gate2InventoryRoot: gate2Inventory.artifactRoot, + gate3StackRoot: gate3Stack.artifactRoot, + sourceSafeMetadataOnly: rows.every((item) => + item.protectedSourceVisible === false + && item.rawPromptTextSerialized === false + && item.rawProviderResponseSerialized === false + && item.credentialsSerialized === false + && item.unpaidAssetPackSourceVisible === false, + ), + knownCarryForwardGapIds: [ + 'gate5-streams-benchmark-linked-inference-telemetry-with-disclosure-tiers', + 'gate6-hardens-read-need-comprehension-prompt-return-types', + 'gate7-hardens-read-fits-finding-query-prompt-benchmarking', + 'gate9-applies-benchmark-report-to-conversation-tool-prompt-parity', + ], + }; + + const artifactRoot = `v38-prompt-benchmark-report:${digest( + JSON.stringify({ + rows: rows.map((item) => item.rowRoot), + fixtures: fixtureInputs.map((item) => item.fixtureRoot), + predicates: passedPredicateIds, + totals: { + promptPartDocCommentCount: coverage.promptPartDocCommentCount, + promptDocCommentCount: coverage.promptDocCommentCount, + benchmarkDefinitionCount: coverage.benchmarkDefinitionCount, + }, + }), + )}`; + + return { + artifactId: 'v38-prompt-benchmark-report', + schemaId: V38_PROMPT_BENCHMARK_REPORT_SCHEMA_ID, + version: V38_PROMPT_BENCHMARK_REPORT_VERSION, + currentTarget: V38_PROMPT_BENCHMARK_REPORT_CURRENT_TARGET, + generatedAt, + artifactPath: V38_PROMPT_BENCHMARK_REPORT_ARTIFACT_PATH, + sourceSafetyVerdict: V38_PROMPT_BENCHMARK_REPORT_SOURCE_SAFETY_VERDICT, + subjectKindIds: [...V38_PROMPT_BENCHMARK_SUBJECT_KIND_IDS], + metricIds: [...V38_PROMPT_BENCHMARK_METRIC_IDS], + disclosureTierIds: [...V38_PROMPT_BENCHMARK_DISCLOSURE_TIER_IDS], + forbiddenPayloadClasses: [...FORBIDDEN_PAYLOAD_CLASSES], + rows, + fixtureInputs, + expectedTypedOutputQualityIds, + sourcePredicateResults, + sourceEvidence, + sourceStatsByRow, + coverage, + failures, + passed: + failures.length === 0 + && failedPredicateIds.length === 0 + && gate2Inventory.passed + && gate3Stack.passed + && coverage.rowCount === 7 + && coverage.promptPartSuiteCount === 3 + && coverage.completePromptSuiteCount === 3 + && coverage.benchmarkInfrastructureSuiteCount === 1 + && coverage.fixtureCount >= 12 + && coverage.expectedTypedOutputQualityCount >= 24 + && coverage.promptPartDocCommentCount > 0 + && coverage.promptDocCommentCount > 0 + && coverage.benchmarkDefinitionCount > 0 + && coverage.promptPartExportCount > 0 + && coverage.promptConstructionCount > 0 + && coverage.sourceSafeMetadataOnly + && coverage.legacySourceRoots === false, + artifactRoot, + }; +} diff --git a/packages/protocol/src/index.d.ts b/packages/protocol/src/index.d.ts index 5f1a0a335..c73c7f8f2 100644 --- a/packages/protocol/src/index.d.ts +++ b/packages/protocol/src/index.d.ts @@ -174,6 +174,16 @@ export const V38_FAILSAFE_STAGE_IDS: readonly string[]; export const V38_THRICIFIED_GENERATION_STAGE_IDS: readonly string[]; export const V38_PTRR_FAILSAFE_THRICIFIED_STACK_ROWS: readonly Record[]; export function buildV38PtrrFailsafeThricifiedStack(input?: Record): BitcodeProtocolReport; +export const V38_PROMPT_BENCHMARK_REPORT_ARTIFACT_PATH: string; +export const V38_PROMPT_BENCHMARK_REPORT_CURRENT_TARGET: string; +export const V38_PROMPT_BENCHMARK_REPORT_SCHEMA_ID: string; +export const V38_PROMPT_BENCHMARK_REPORT_VERSION: string; +export const V38_PROMPT_BENCHMARK_REPORT_SOURCE_SAFETY_VERDICT: string; +export const V38_PROMPT_BENCHMARK_SUBJECT_KIND_IDS: readonly string[]; +export const V38_PROMPT_BENCHMARK_METRIC_IDS: readonly string[]; +export const V38_PROMPT_BENCHMARK_DISCLOSURE_TIER_IDS: readonly string[]; +export const V38_PROMPT_BENCHMARK_ROWS: readonly Record[]; +export function buildV38PromptBenchmarkReport(input?: Record): BitcodeProtocolReport; export const EXCHANGE_INTENT_ORDER_CONTRACTS_ARTIFACT_PATH: string; export const EXCHANGE_INTENT_ORDER_CONTRACTS_CURRENT_TARGET: string; export const EXCHANGE_INTENT_ORDER_CONTRACTS_SCHEMA_ID: string; diff --git a/packages/protocol/src/index.js b/packages/protocol/src/index.js index bbc40bd4d..ba456d0e5 100644 --- a/packages/protocol/src/index.js +++ b/packages/protocol/src/index.js @@ -160,6 +160,18 @@ export { V38_THRICIFIED_GENERATION_STAGE_IDS, buildV38PtrrFailsafeThricifiedStack } from './canonical/ptrr-failsafe-thricified-stack.js'; +export { + V38_PROMPT_BENCHMARK_DISCLOSURE_TIER_IDS, + V38_PROMPT_BENCHMARK_METRIC_IDS, + V38_PROMPT_BENCHMARK_REPORT_ARTIFACT_PATH, + V38_PROMPT_BENCHMARK_REPORT_CURRENT_TARGET, + V38_PROMPT_BENCHMARK_REPORT_SCHEMA_ID, + V38_PROMPT_BENCHMARK_REPORT_SOURCE_SAFETY_VERDICT, + V38_PROMPT_BENCHMARK_REPORT_VERSION, + V38_PROMPT_BENCHMARK_ROWS, + V38_PROMPT_BENCHMARK_SUBJECT_KIND_IDS, + buildV38PromptBenchmarkReport +} from './canonical/prompt-benchmark-report.js'; export { EXCHANGE_INTENT_ACTION_KINDS, EXCHANGE_INTENT_ORDER_CONTRACTS_ARTIFACT_PATH, diff --git a/packages/protocol/test/v38-prompt-benchmark-report.test.js b/packages/protocol/test/v38-prompt-benchmark-report.test.js new file mode 100644 index 000000000..8a588e8f1 --- /dev/null +++ b/packages/protocol/test/v38-prompt-benchmark-report.test.js @@ -0,0 +1,110 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { test } from 'node:test'; + +import { + V38_PROMPT_BENCHMARK_DISCLOSURE_TIER_IDS, + V38_PROMPT_BENCHMARK_METRIC_IDS, + V38_PROMPT_BENCHMARK_REPORT_ARTIFACT_PATH, + V38_PROMPT_BENCHMARK_REPORT_SOURCE_SAFETY_VERDICT, + V38_PROMPT_BENCHMARK_ROWS, + V38_PROMPT_BENCHMARK_SUBJECT_KIND_IDS, + buildV38PromptBenchmarkReport, +} from '../src/index.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(__dirname, '..', '..', '..'); + +test('builds source-safe V38 PromptPart and Prompt benchmark report', () => { + const report = buildV38PromptBenchmarkReport({ + generatedAt: '2026-05-24T00:00:00.000Z', + repoRoot, + }); + + assert.equal(report.artifactId, 'v38-prompt-benchmark-report'); + assert.equal(report.schemaId, 'bitcode.v38.promptBenchmarkReport.v1'); + assert.equal(report.version, 'V38'); + assert.equal(report.currentTarget, 'V37'); + assert.equal(report.passed, true); + assert.equal(report.artifactPath, V38_PROMPT_BENCHMARK_REPORT_ARTIFACT_PATH); + assert.equal(report.sourceSafetyVerdict, V38_PROMPT_BENCHMARK_REPORT_SOURCE_SAFETY_VERDICT); + assert.deepEqual(report.subjectKindIds, [...V38_PROMPT_BENCHMARK_SUBJECT_KIND_IDS]); + assert.deepEqual(report.metricIds, [...V38_PROMPT_BENCHMARK_METRIC_IDS]); + assert.deepEqual(report.disclosureTierIds, [...V38_PROMPT_BENCHMARK_DISCLOSURE_TIER_IDS]); + assert.equal(report.rows.length, V38_PROMPT_BENCHMARK_ROWS.length); + assert.equal(report.failures.length, 0); + assert.match(report.artifactRoot, /^v38-prompt-benchmark-report:[a-f0-9]{24}$/u); +}); + +test('covers active Reading, Conversation, and tool-definition benchmark suites', () => { + const report = buildV38PromptBenchmarkReport({ + generatedAt: '2026-05-24T00:00:00.000Z', + repoRoot, + }); + + assert.equal(report.coverage.rowCount, 7); + assert.equal(report.coverage.promptPartSuiteCount, 3); + assert.equal(report.coverage.completePromptSuiteCount, 3); + assert.equal(report.coverage.benchmarkInfrastructureSuiteCount, 1); + assert.equal(report.coverage.fixtureCount >= 12, true); + assert.equal(report.coverage.expectedTypedOutputQualityCount >= 24, true); + assert.equal(report.coverage.gate2InventoryRoot.length > 0, true); + assert.equal(report.coverage.gate3StackRoot.length > 0, true); + + const rowIds = report.rows.map((row) => row.rowId); + for (const requiredRowId of [ + 'benchmark:runner-and-doc-comment-infrastructure', + 'promptpart:generic-ptrr-failsafe-thricified-foundation', + 'promptpart:read-need-comprehension-specific', + 'promptpart:read-fits-finding-specific', + 'prompt:reading-pipeline-agent-prompts', + 'prompt:conversation-system-prompt', + 'prompt:tool-definition-doc-code-prompts', + ]) { + assert.equal(rowIds.includes(requiredRowId), true, `missing ${requiredRowId}`); + } +}); + +test('proves source metadata and benchmark predicates without serializing protected prompt payloads', () => { + const report = buildV38PromptBenchmarkReport({ + generatedAt: '2026-05-24T00:00:00.000Z', + repoRoot, + }); + + assert.equal(report.coverage.requiredPredicateCount, report.coverage.passedPredicateCount); + assert.deepEqual(report.coverage.failedPredicateIds, []); + assert.equal(report.coverage.promptPartDocCommentCount > 0, true); + assert.equal(report.coverage.promptDocCommentCount > 0, true); + assert.equal(report.coverage.benchmarkDefinitionCount > 0, true); + assert.equal(report.coverage.promptPartExportCount > 0, true); + assert.equal(report.coverage.promptConstructionCount > 0, true); + assert.equal(report.coverage.sourceSafeMetadataOnly, true); + assert.equal(report.coverage.legacySourceRoots, false); + + for (const row of report.rows) { + assert.match(row.rowRoot, /^v38-prompt-benchmark-row:[a-f0-9]{24}$/u); + assert.equal(row.sourceSafetyClass, 'source_safe_prompt_benchmark_metadata'); + assert.equal(row.protectedSourceVisible, false); + assert.equal(row.rawPromptTextSerialized, false); + assert.equal(row.rawProviderResponseSerialized, false); + assert.equal(row.credentialsSerialized, false); + assert.equal(row.unpaidAssetPackSourceVisible, false); + assert.equal(row.sourceRootsPresent, true); + assert.equal(Array.isArray(row.fixtureInputs), true); + assert.equal(row.fixtureInputs.every((fixture) => /^v38-prompt-benchmark-fixture:[a-f0-9]{24}$/u.test(fixture.fixtureRoot)), true); + } + + const predicateIds = report.sourcePredicateResults.map((result) => result.id); + for (const requiredPredicateId of [ + 'benchmark.runner-defines-promptpart-benchmark', + 'benchmark.runner-defines-complete-prompt-benchmark', + 'promptpart.read-need-contract-names-pipeline', + 'promptpart.read-fits-depository-search-names-tools', + 'prompt.reading-contract-has-template-ids', + 'prompt.conversation-requires-hierarchy', + 'prompt.tool-format-usable-tools-present', + ]) { + assert.equal(predicateIds.includes(requiredPredicateId), true, `missing ${requiredPredicateId}`); + } +}); diff --git a/scripts/check-v38-gate4-prompt-benchmark-report.mjs b/scripts/check-v38-gate4-prompt-benchmark-report.mjs new file mode 100644 index 000000000..b6d966407 --- /dev/null +++ b/scripts/check-v38-gate4-prompt-benchmark-report.mjs @@ -0,0 +1,293 @@ +#!/usr/bin/env node + +import { execFileSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const defaultRepoRoot = path.resolve(__dirname, '..'); +const ARTIFACT_PATH = '.bitcode/v38-prompt-benchmark-report.json'; + +const REQUIRED_ROW_IDS = [ + 'benchmark:runner-and-doc-comment-infrastructure', + 'promptpart:generic-ptrr-failsafe-thricified-foundation', + 'promptpart:read-need-comprehension-specific', + 'promptpart:read-fits-finding-specific', + 'prompt:reading-pipeline-agent-prompts', + 'prompt:conversation-system-prompt', + 'prompt:tool-definition-doc-code-prompts', +]; + +const REQUIRED_METRIC_IDS = [ + 'intent_alignment', + 'semantic_clarity', + 'token_efficiency', + 'model_stability', + 'task_success', + 'response_quality', + 'schema_conformance', + 'source_boundary_preservation', +]; + +const SECRET_MARKERS = [ + `${['sk', 'proj'].join('-')}-`, + `${['sb', 'secret'].join('_')}__`, + ['service', 'role'].join('_'), + ['eyJhbGciOi', 'JIUzI1Ni'].join(''), + ['OPENAI', 'API', 'KEY'].join('_'), + ['SUPABASE', 'SERVICE', 'ROLE'].join('_'), + ['VERCEL', 'TOKEN'].join('_'), + ['VERCEL', 'OIDC', 'TOKEN'].join('_'), + ['PRIVATE', 'KEY'].join('_'), +]; + +function read(root, relativePath) { + return readFileSync(path.join(root, relativePath), 'utf8'); +} + +function fileExists(root, relativePath) { + return existsSync(path.join(root, relativePath)); +} + +function git(root, args) { + return execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim(); +} + +function run(root, command, args) { + return execFileSync(command, args, { + cwd: root, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }).trim(); +} + +function assertCheck(failures, condition, message) { + if (!condition) failures.push(message); +} + +function includesAll(values, requiredValues) { + return requiredValues.every((value) => values.includes(value)); +} + +function parseArgs(argv) { + const args = { + skipBranchCheck: false, + repoRoot: defaultRepoRoot, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--skip-branch-check') args.skipBranchCheck = true; + else if (arg === '--repo-root') args.repoRoot = path.resolve(argv[++index]); + else if (arg === '--help' || arg === '-h') args.help = true; + else throw new Error(`Unknown argument ${arg}`); + } + + return args; +} + +function printHelp() { + process.stdout.write( + [ + 'Usage: node scripts/check-v38-gate4-prompt-benchmark-report.mjs [--skip-branch-check] [--repo-root ]', + '', + 'Checks V38 Gate 4 source-safe PromptPart and Prompt benchmark report artifacts, tests, docs, and workflow wiring.', + ].join('\n'), + ); + process.stdout.write('\n'); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + printHelp(); + return; + } + + const root = args.repoRoot; + const failures = []; + const pointer = read(root, 'BITCODE_SPEC.txt').trim(); + + assertCheck( + failures, + pointer === 'V37', + `BITCODE_SPEC.txt must remain V37 during V38 gate work. Observed ${pointer || 'empty'}.`, + ); + + if (!args.skipBranchCheck) { + const branch = git(root, ['branch', '--show-current']); + assertCheck( + failures, + branch === 'version/v38' || /^v38\/gate-(?:[4-9]|10|11)-[a-z0-9][a-z0-9-]*$/u.test(branch), + `V38 Gate 4+ work must occur on version/v38 or v38/gate-4..11-* branches. Observed ${branch || 'detached HEAD'}.`, + ); + } + + const requiredFiles = [ + ARTIFACT_PATH, + '.bitcode/v38-inference-surface-inventory.json', + '.bitcode/v38-ptrr-failsafe-thricified-stack.json', + 'packages/protocol/src/canonical/prompt-benchmark-report.js', + 'packages/protocol/src/canonical/inference-surface-inventory.js', + 'packages/protocol/src/canonical/ptrr-failsafe-thricified-stack.js', + 'packages/protocol/src/index.js', + 'packages/protocol/src/index.d.ts', + 'packages/protocol/test/v38-prompt-benchmark-report.test.js', + 'scripts/generate-v38-prompt-benchmark-report.mjs', + 'scripts/check-v38-gate4-prompt-benchmark-report.mjs', + 'packages/prompts/src/benchmarking/runner.ts', + 'packages/prompts/src/benchmarking/types.ts', + 'packages/prompts/src/benchmarking/README.md', + 'packages/prompts/src/developing/doc-comment-developing.ts', + 'packages/pipelines/asset-pack/src/reading-pipeline-contract.ts', + 'packages/conversations-generics/src/prompts/BitcodeTerminalConversationSystemPrompt.ts', + 'packages/tools-generics/src/doc-code-tool/DocCodeToolPrompt.ts', + 'packages/tools-generics/src/doc-code-tool/formatUsableTools.ts', + 'BITCODE_SPEC_V38.md', + 'BITCODE_SPEC_V38_DELTA.md', + 'BITCODE_SPEC_V38_NOTES.md', + 'BITCODE_SPEC_V38_PARITY_MATRIX.md', + 'SPECIFICATIONS_ROADMAP.md', + 'README.md', + 'packages/protocol/README.md', + 'package.json', + '.github/workflows/bitcode-gate-quality.yml', + '.github/workflows/bitcode-canon-quality.yml', + ]; + + for (const relativePath of requiredFiles) { + assertCheck(failures, fileExists(root, relativePath), `Missing V38 Gate 4 file: ${relativePath}`); + } + + if (failures.length === 0) { + try { + run(root, 'node', ['scripts/generate-v38-prompt-benchmark-report.mjs', '--check']); + } catch (error) { + failures.push(`V38 prompt benchmark report artifact check failed: ${error.stderr || error.message}`); + } + } + + if (failures.length === 0) { + try { + run(root, 'node', ['--test', '--test-force-exit', 'packages/protocol/test/v38-prompt-benchmark-report.test.js']); + } catch (error) { + failures.push(`V38 prompt benchmark report protocol test failed: ${error.stderr || error.message}`); + } + } + + const serializedArtifact = fileExists(root, ARTIFACT_PATH) ? read(root, ARTIFACT_PATH) : ''; + for (const marker of SECRET_MARKERS) { + assertCheck(failures, !serializedArtifact.includes(marker), `V38 prompt benchmark artifact must not contain secret marker ${marker}.`); + } + + const artifact = serializedArtifact ? JSON.parse(serializedArtifact) : null; + if (artifact) { + assertCheck(failures, artifact.artifactId === 'v38-prompt-benchmark-report', 'Prompt benchmark artifactId must match.'); + assertCheck(failures, artifact.schemaId === 'bitcode.v38.promptBenchmarkReport.v1', 'Prompt benchmark schemaId must match.'); + assertCheck(failures, artifact.version === 'V38' && artifact.currentTarget === 'V37', 'Prompt benchmark report must bind V38 over active V37.'); + assertCheck(failures, artifact.passed === true, 'Prompt benchmark report artifact must pass.'); + assertCheck( + failures, + artifact.sourceSafetyVerdict === 'source-safe-prompt-benchmark-metadata', + 'Prompt benchmark report must be source-safe metadata.', + ); + assertCheck(failures, includesAll(artifact.rows.map((row) => row.rowId), REQUIRED_ROW_IDS), 'Prompt benchmark rows must cover every required row id.'); + assertCheck(failures, includesAll(artifact.metricIds, REQUIRED_METRIC_IDS), 'Prompt benchmark metrics must include every required metric id.'); + assertCheck(failures, artifact.coverage.rowCount === 7, 'Prompt benchmark report must have 7 rows.'); + assertCheck(failures, artifact.coverage.promptPartSuiteCount === 3, 'Prompt benchmark report must cover 3 PromptPart suites.'); + assertCheck(failures, artifact.coverage.completePromptSuiteCount === 3, 'Prompt benchmark report must cover 3 complete Prompt suites.'); + assertCheck(failures, artifact.coverage.benchmarkInfrastructureSuiteCount === 1, 'Prompt benchmark report must cover benchmark infrastructure.'); + assertCheck(failures, artifact.coverage.fixtureCount >= 12, 'Prompt benchmark report must include at least 12 fixtures.'); + assertCheck(failures, artifact.coverage.expectedTypedOutputQualityCount >= 24, 'Prompt benchmark report must include typed-output quality expectations.'); + assertCheck(failures, artifact.coverage.requiredPredicateCount === artifact.coverage.passedPredicateCount, 'Prompt benchmark predicates must all pass.'); + assertCheck(failures, artifact.coverage.failedPredicateIds.length === 0, 'Prompt benchmark report must have no failed predicates.'); + assertCheck(failures, artifact.coverage.promptPartDocCommentCount > 0, 'Prompt benchmark report must see PromptPart doc-comments.'); + assertCheck(failures, artifact.coverage.promptDocCommentCount > 0, 'Prompt benchmark report must see Prompt doc-comments.'); + assertCheck(failures, artifact.coverage.benchmarkDefinitionCount > 0, 'Prompt benchmark report must see benchmark definitions.'); + assertCheck(failures, artifact.coverage.promptPartExportCount > 0, 'Prompt benchmark report must see PromptPart exports.'); + assertCheck(failures, artifact.coverage.promptConstructionCount > 0, 'Prompt benchmark report must see Prompt constructions.'); + assertCheck(failures, artifact.coverage.sourceSafeMetadataOnly === true, 'Prompt benchmark report must be metadata-only.'); + assertCheck(failures, artifact.coverage.legacySourceRoots === false, 'Prompt benchmark report must not point at _legacy source roots.'); + assertCheck(failures, artifact.coverage.gate2InventoryRoot.startsWith('v38-inference-surface-inventory:'), 'Prompt benchmark report must bind Gate 2 inventory root.'); + assertCheck(failures, artifact.coverage.gate3StackRoot.startsWith('v38-ptrr-failsafe-thricified-stack:'), 'Prompt benchmark report must bind Gate 3 stack root.'); + assertCheck( + failures, + artifact.rows.every((row) => /^v38-prompt-benchmark-row:[a-f0-9]{24}$/u.test(row.rowRoot)), + 'Prompt benchmark rows must have deterministic row roots.', + ); + assertCheck( + failures, + artifact.fixtureInputs.every((fixture) => /^v38-prompt-benchmark-fixture:[a-f0-9]{24}$/u.test(fixture.fixtureRoot)), + 'Prompt benchmark fixtures must have deterministic fixture roots.', + ); + } + + const spec = read(root, 'BITCODE_SPEC_V38.md'); + const delta = read(root, 'BITCODE_SPEC_V38_DELTA.md'); + const notes = read(root, 'BITCODE_SPEC_V38_NOTES.md'); + const parity = read(root, 'BITCODE_SPEC_V38_PARITY_MATRIX.md'); + const roadmap = read(root, 'SPECIFICATIONS_ROADMAP.md'); + const readme = read(root, 'README.md'); + const protocolReadme = read(root, 'packages/protocol/README.md'); + const packageJson = read(root, 'package.json'); + const gateWorkflow = read(root, '.github/workflows/bitcode-gate-quality.yml'); + const canonWorkflow = read(root, '.github/workflows/bitcode-canon-quality.yml'); + const source = read(root, 'packages/protocol/src/canonical/prompt-benchmark-report.js'); + const index = read(root, 'packages/protocol/src/index.js'); + const typeDefs = read(root, 'packages/protocol/src/index.d.ts'); + const test = read(root, 'packages/protocol/test/v38-prompt-benchmark-report.test.js'); + + for (const doc of [spec, delta, notes, parity, readme, protocolReadme]) { + assertCheck(failures, doc.includes(ARTIFACT_PATH), `V38 docs must mention ${ARTIFACT_PATH}.`); + assertCheck(failures, doc.includes('V38PromptBenchmarkReport'), 'V38 docs must name V38PromptBenchmarkReport.'); + assertCheck(failures, doc.includes('source-safe-prompt-benchmark-metadata'), 'V38 docs must name prompt benchmark source safety verdict.'); + assertCheck(failures, doc.includes('PromptPart'), 'V38 docs must name PromptPart.'); + assertCheck(failures, doc.includes('Prompt benchmarking'), 'V38 docs must name Prompt benchmarking.'); + } + + assertCheck(failures, parity.includes('| PromptPart and Prompt benchmarking | Gate 4 |') && parity.includes('| closed |'), 'V38 parity must close the Gate 4 matrix row.'); + assertCheck(failures, parity.includes('## Gate 4 Parity') && parity.includes('closed'), 'V38 parity must include closed Gate 4 parity.'); + assertCheck( + failures, + /Current working gate: V38 Gate (?:5|6|7|8|9|10|11)\b/u.test(roadmap), + 'Roadmap must advance past V38 Gate 4 after this gate closes.', + ); + assertCheck(failures, roadmap.includes('V38 Gate 4 closure anchor'), 'Roadmap must include a V38 Gate 4 closure anchor.'); + assertCheck(failures, packageJson.includes('"generate:v38-prompt-benchmark-report"'), 'package.json must expose the Gate 4 generator.'); + assertCheck(failures, packageJson.includes('"check:v38-prompt-benchmark-report"'), 'package.json must expose the Gate 4 artifact check.'); + assertCheck(failures, packageJson.includes('"check:v38-gate4"'), 'package.json must expose check:v38-gate4.'); + assertCheck(failures, gateWorkflow.includes('check-v38-gate4-prompt-benchmark-report.mjs'), 'Gate workflow must run the V38 Gate 4 checker.'); + assertCheck(failures, canonWorkflow.includes('check-v38-gate4-prompt-benchmark-report.mjs'), 'Canon workflow must run the V38 Gate 4 checker.'); + assertCheck(failures, gateWorkflow.includes('v38-prompt-benchmark-report.test.js'), 'Gate workflow must run the V38 Gate 4 protocol test.'); + + for (const phrase of [ + 'buildV38PromptBenchmarkReport', + 'V38_PROMPT_BENCHMARK_ROWS', + 'fixtureInputs', + 'expectedTypedOutputQualityIds', + 'rawPromptTextSerialized', + 'sourceSafeMetadataOnly', + ]) { + assertCheck(failures, source.includes(phrase), `Gate 4 source must include ${phrase}.`); + } + + for (const phrase of ['buildV38PromptBenchmarkReport', 'fixtureCount', 'sourceSafeMetadataOnly']) { + assertCheck(failures, test.includes(phrase), `Gate 4 test must include ${phrase}.`); + } + + assertCheck(failures, index.includes('buildV38PromptBenchmarkReport'), 'Protocol index must export buildV38PromptBenchmarkReport.'); + assertCheck(failures, typeDefs.includes('buildV38PromptBenchmarkReport'), 'Protocol type declarations must export buildV38PromptBenchmarkReport.'); + + if (failures.length > 0) { + process.stderr.write('V38 Gate 4 PromptPart and Prompt benchmark check failed:\n'); + for (const failure of failures) process.stderr.write(`- ${failure}\n`); + process.exitCode = 1; + return; + } + + process.stdout.write('V38 Gate 4 PromptPart and Prompt benchmarking ok rows=7 fixtures>=12 source-safe=true\n'); +} + +main(); diff --git a/scripts/generate-v38-prompt-benchmark-report.mjs b/scripts/generate-v38-prompt-benchmark-report.mjs new file mode 100644 index 000000000..a151bb888 --- /dev/null +++ b/scripts/generate-v38-prompt-benchmark-report.mjs @@ -0,0 +1,60 @@ +#!/usr/bin/env node + +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + V38_PROMPT_BENCHMARK_REPORT_ARTIFACT_PATH, + buildV38PromptBenchmarkReport, +} from '../packages/protocol/src/index.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const repoRoot = path.resolve(__dirname, '..'); +const GENERATED_AT = '2026-05-24T00:00:00.000Z'; + +function parseArgs(argv) { + return { + check: argv.includes('--check'), + write: argv.includes('--write') || !argv.includes('--check'), + }; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const artifact = buildV38PromptBenchmarkReport({ + generatedAt: GENERATED_AT, + repoRoot, + }); + const outputPath = path.join(repoRoot, V38_PROMPT_BENCHMARK_REPORT_ARTIFACT_PATH); + const serialized = `${JSON.stringify(artifact, null, 2)}\n`; + + if (args.check) { + const current = readFileSync(outputPath, 'utf8'); + if (current !== serialized) { + process.stderr.write( + `${V38_PROMPT_BENCHMARK_REPORT_ARTIFACT_PATH} is stale. Run pnpm run generate:v38-prompt-benchmark-report.\n`, + ); + process.exitCode = 1; + return; + } + if (!artifact.passed) { + process.stderr.write(`V38 prompt benchmark report failed:\n${artifact.failures.map((failure) => `- ${failure}`).join('\n')}\n`); + process.exitCode = 1; + return; + } + process.stdout.write( + `V38 prompt benchmark report ok rows=${artifact.rows.length} fixtures=${artifact.coverage.fixtureCount} root=${artifact.artifactRoot}\n`, + ); + return; + } + + mkdirSync(path.dirname(outputPath), { recursive: true }); + writeFileSync(outputPath, serialized); + process.stdout.write( + `Wrote ${V38_PROMPT_BENCHMARK_REPORT_ARTIFACT_PATH} rows=${artifact.rows.length} fixtures=${artifact.coverage.fixtureCount} root=${artifact.artifactRoot}\n`, + ); +} + +main();