Skip to content

Commit d31b4f2

Browse files
geraldarthurdavisgeraldarthurdavis
committed
Stabilize V41 Reading benchmark artifact
Ignore generated JavaScript siblings when TypeScript prompt sources exist so the Gate 4 baseline artifact is deterministic between local workspaces and clean CI checkouts. Validation: pnpm run generate:v41-reading-prompt-benchmark-baselines; pnpm run check:v41-reading-prompt-benchmark-baselines; pnpm run check:v41-gate4; node --test --test-force-exit packages/protocol/test/v41-reading-prompt-benchmark-baselines.test.js; git diff --check.
1 parent de38e00 commit d31b4f2

2 files changed

Lines changed: 46 additions & 51 deletions

File tree

‎.bitcode/v41-reading-prompt-benchmark-baselines.json‎

Lines changed: 42 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -587,20 +587,18 @@
587587
"sourceStats": {
588588
"baselineId": "readfits-query-synthesis-baseline",
589589
"sourceRootsPresent": true,
590-
"sourceFileCount": 15,
590+
"sourceFileCount": 10,
591591
"sourceFileSample": [
592-
"packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.js",
593592
"packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.ts",
594-
"packages/generic-tools/simple-system-text-search/src/prompts/SimpleSystemTextSearchDocCodeToolPrompt.js",
595593
"packages/generic-tools/simple-system-text-search/src/prompts/SimpleSystemTextSearchDocCodeToolPrompt.ts",
596-
"packages/generic-tools/web-search/src/prompts/GetContentsDocCodeToolPrompt.js",
597594
"packages/generic-tools/web-search/src/prompts/GetContentsDocCodeToolPrompt.ts",
598-
"packages/generic-tools/web-search/src/prompts/MultiProviderSearchDocCodeToolPrompt.js",
599595
"packages/generic-tools/web-search/src/prompts/MultiProviderSearchDocCodeToolPrompt.ts",
600-
"packages/generic-tools/web-search/src/prompts/WebSearchDocCodeToolPrompt.js",
601596
"packages/generic-tools/web-search/src/prompts/WebSearchDocCodeToolPrompt.ts",
602597
"packages/pipelines/asset-pack/src/depository-search.ts",
603-
"packages/pipelines/asset-pack/src/depository-supply-index.ts"
598+
"packages/pipelines/asset-pack/src/depository-supply-index.ts",
599+
"packages/pipelines/asset-pack/src/embedding-config.ts",
600+
"packages/pipelines/asset-pack/src/read-fits-finding-runtime.ts",
601+
"packages/pipelines/asset-pack/src/reading-pipeline-contract.ts"
604602
],
605603
"promptTemplateIdCount": 6,
606604
"interpolatedContextKeyCount": 6,
@@ -895,20 +893,20 @@
895893
"sourceStats": {
896894
"baselineId": "readfits-assetpack-context-synthesis-baseline",
897895
"sourceRootsPresent": true,
898-
"sourceFileCount": 35,
896+
"sourceFileCount": 27,
899897
"sourceFileSample": [
900-
"packages/generic-tools/vcs/src/prompts/CreateCommentDocCodeToolPrompt.js",
901898
"packages/generic-tools/vcs/src/prompts/CreateCommentDocCodeToolPrompt.ts",
902-
"packages/generic-tools/vcs/src/prompts/CreateIssueDocCodeToolPrompt.js",
903899
"packages/generic-tools/vcs/src/prompts/CreateIssueDocCodeToolPrompt.ts",
904-
"packages/generic-tools/vcs/src/prompts/CreateOrUpdateFileDocCodeToolPrompt.js",
905900
"packages/generic-tools/vcs/src/prompts/CreateOrUpdateFileDocCodeToolPrompt.ts",
906-
"packages/generic-tools/vcs/src/prompts/CreatePullRequestDocCodeToolPrompt.js",
907901
"packages/generic-tools/vcs/src/prompts/CreatePullRequestDocCodeToolPrompt.ts",
908-
"packages/generic-tools/vcs/src/prompts/GetFileContentDocCodeToolPrompt.js",
909902
"packages/generic-tools/vcs/src/prompts/GetFileContentDocCodeToolPrompt.ts",
910-
"packages/generic-tools/vcs/src/prompts/ListBranchesDocCodeToolPrompt.js",
911-
"packages/generic-tools/vcs/src/prompts/ListBranchesDocCodeToolPrompt.ts"
903+
"packages/generic-tools/vcs/src/prompts/ListBranchesDocCodeToolPrompt.ts",
904+
"packages/generic-tools/vcs/src/prompts/ListRepositoriesDocCodeToolPrompt.ts",
905+
"packages/generic-tools/vcs/src/prompts/index.ts",
906+
"packages/pipelines/asset-pack/src/agents/implementation/read-fits-finding-synthesis-asset-pack-synthesis-agent.ts",
907+
"packages/pipelines/asset-pack/src/agents/prompts/analyze-codebase-prompt.ts",
908+
"packages/pipelines/asset-pack/src/agents/prompts/analyze-parallel-prompt.ts",
909+
"packages/pipelines/asset-pack/src/agents/prompts/assess-complexity-prompt.ts"
912910
],
913911
"promptTemplateIdCount": 7,
914912
"interpolatedContextKeyCount": 6,
@@ -925,7 +923,7 @@
925923
"zodSchemaCount": 16,
926924
"telemetryCount": 90,
927925
"sourceSafetyCount": 5,
928-
"benchmarkDefinitionCount": 23,
926+
"benchmarkDefinitionCount": 22,
929927
"queryPlanCount": 4,
930928
"embeddingPolicyCount": 9,
931929
"rankingCount": 12,
@@ -1195,20 +1193,18 @@
11951193
"sourceStats": {
11961194
"baselineId": "settlement-rights-delivery-baseline",
11971195
"sourceRootsPresent": true,
1198-
"sourceFileCount": 18,
1196+
"sourceFileCount": 10,
11991197
"sourceFileSample": [
1200-
"packages/generic-tools/vcs/src/prompts/CreateCommentDocCodeToolPrompt.js",
12011198
"packages/generic-tools/vcs/src/prompts/CreateCommentDocCodeToolPrompt.ts",
1202-
"packages/generic-tools/vcs/src/prompts/CreateIssueDocCodeToolPrompt.js",
12031199
"packages/generic-tools/vcs/src/prompts/CreateIssueDocCodeToolPrompt.ts",
1204-
"packages/generic-tools/vcs/src/prompts/CreateOrUpdateFileDocCodeToolPrompt.js",
12051200
"packages/generic-tools/vcs/src/prompts/CreateOrUpdateFileDocCodeToolPrompt.ts",
1206-
"packages/generic-tools/vcs/src/prompts/CreatePullRequestDocCodeToolPrompt.js",
12071201
"packages/generic-tools/vcs/src/prompts/CreatePullRequestDocCodeToolPrompt.ts",
1208-
"packages/generic-tools/vcs/src/prompts/GetFileContentDocCodeToolPrompt.js",
12091202
"packages/generic-tools/vcs/src/prompts/GetFileContentDocCodeToolPrompt.ts",
1210-
"packages/generic-tools/vcs/src/prompts/ListBranchesDocCodeToolPrompt.js",
1211-
"packages/generic-tools/vcs/src/prompts/ListBranchesDocCodeToolPrompt.ts"
1203+
"packages/generic-tools/vcs/src/prompts/ListBranchesDocCodeToolPrompt.ts",
1204+
"packages/generic-tools/vcs/src/prompts/ListRepositoriesDocCodeToolPrompt.ts",
1205+
"packages/generic-tools/vcs/src/prompts/index.ts",
1206+
"packages/pipelines/asset-pack/src/asset-pack-settlement-rights-delivery.ts",
1207+
"packages/pipelines/asset-pack/src/reading-pipeline-contract.ts"
12121208
],
12131209
"promptTemplateIdCount": 6,
12141210
"interpolatedContextKeyCount": 6,
@@ -1225,7 +1221,7 @@
12251221
"zodSchemaCount": 2,
12261222
"telemetryCount": 90,
12271223
"sourceSafetyCount": 39,
1228-
"benchmarkDefinitionCount": 2,
1224+
"benchmarkDefinitionCount": 1,
12291225
"queryPlanCount": 1,
12301226
"embeddingPolicyCount": 5,
12311227
"rankingCount": 9,
@@ -1668,20 +1664,18 @@
16681664
"readfits-query-synthesis-baseline": {
16691665
"baselineId": "readfits-query-synthesis-baseline",
16701666
"sourceRootsPresent": true,
1671-
"sourceFileCount": 15,
1667+
"sourceFileCount": 10,
16721668
"sourceFileSample": [
1673-
"packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.js",
16741669
"packages/generic-tools/simple-system-text-search/src/prompts/BitcodeRepositoryEvidenceSearchDocCodeToolPrompt.ts",
1675-
"packages/generic-tools/simple-system-text-search/src/prompts/SimpleSystemTextSearchDocCodeToolPrompt.js",
16761670
"packages/generic-tools/simple-system-text-search/src/prompts/SimpleSystemTextSearchDocCodeToolPrompt.ts",
1677-
"packages/generic-tools/web-search/src/prompts/GetContentsDocCodeToolPrompt.js",
16781671
"packages/generic-tools/web-search/src/prompts/GetContentsDocCodeToolPrompt.ts",
1679-
"packages/generic-tools/web-search/src/prompts/MultiProviderSearchDocCodeToolPrompt.js",
16801672
"packages/generic-tools/web-search/src/prompts/MultiProviderSearchDocCodeToolPrompt.ts",
1681-
"packages/generic-tools/web-search/src/prompts/WebSearchDocCodeToolPrompt.js",
16821673
"packages/generic-tools/web-search/src/prompts/WebSearchDocCodeToolPrompt.ts",
16831674
"packages/pipelines/asset-pack/src/depository-search.ts",
1684-
"packages/pipelines/asset-pack/src/depository-supply-index.ts"
1675+
"packages/pipelines/asset-pack/src/depository-supply-index.ts",
1676+
"packages/pipelines/asset-pack/src/embedding-config.ts",
1677+
"packages/pipelines/asset-pack/src/read-fits-finding-runtime.ts",
1678+
"packages/pipelines/asset-pack/src/reading-pipeline-contract.ts"
16851679
],
16861680
"promptTemplateIdCount": 6,
16871681
"interpolatedContextKeyCount": 6,
@@ -1744,20 +1738,20 @@
17441738
"readfits-assetpack-context-synthesis-baseline": {
17451739
"baselineId": "readfits-assetpack-context-synthesis-baseline",
17461740
"sourceRootsPresent": true,
1747-
"sourceFileCount": 35,
1741+
"sourceFileCount": 27,
17481742
"sourceFileSample": [
1749-
"packages/generic-tools/vcs/src/prompts/CreateCommentDocCodeToolPrompt.js",
17501743
"packages/generic-tools/vcs/src/prompts/CreateCommentDocCodeToolPrompt.ts",
1751-
"packages/generic-tools/vcs/src/prompts/CreateIssueDocCodeToolPrompt.js",
17521744
"packages/generic-tools/vcs/src/prompts/CreateIssueDocCodeToolPrompt.ts",
1753-
"packages/generic-tools/vcs/src/prompts/CreateOrUpdateFileDocCodeToolPrompt.js",
17541745
"packages/generic-tools/vcs/src/prompts/CreateOrUpdateFileDocCodeToolPrompt.ts",
1755-
"packages/generic-tools/vcs/src/prompts/CreatePullRequestDocCodeToolPrompt.js",
17561746
"packages/generic-tools/vcs/src/prompts/CreatePullRequestDocCodeToolPrompt.ts",
1757-
"packages/generic-tools/vcs/src/prompts/GetFileContentDocCodeToolPrompt.js",
17581747
"packages/generic-tools/vcs/src/prompts/GetFileContentDocCodeToolPrompt.ts",
1759-
"packages/generic-tools/vcs/src/prompts/ListBranchesDocCodeToolPrompt.js",
1760-
"packages/generic-tools/vcs/src/prompts/ListBranchesDocCodeToolPrompt.ts"
1748+
"packages/generic-tools/vcs/src/prompts/ListBranchesDocCodeToolPrompt.ts",
1749+
"packages/generic-tools/vcs/src/prompts/ListRepositoriesDocCodeToolPrompt.ts",
1750+
"packages/generic-tools/vcs/src/prompts/index.ts",
1751+
"packages/pipelines/asset-pack/src/agents/implementation/read-fits-finding-synthesis-asset-pack-synthesis-agent.ts",
1752+
"packages/pipelines/asset-pack/src/agents/prompts/analyze-codebase-prompt.ts",
1753+
"packages/pipelines/asset-pack/src/agents/prompts/analyze-parallel-prompt.ts",
1754+
"packages/pipelines/asset-pack/src/agents/prompts/assess-complexity-prompt.ts"
17611755
],
17621756
"promptTemplateIdCount": 7,
17631757
"interpolatedContextKeyCount": 6,
@@ -1774,7 +1768,7 @@
17741768
"zodSchemaCount": 16,
17751769
"telemetryCount": 90,
17761770
"sourceSafetyCount": 5,
1777-
"benchmarkDefinitionCount": 23,
1771+
"benchmarkDefinitionCount": 22,
17781772
"queryPlanCount": 4,
17791773
"embeddingPolicyCount": 9,
17801774
"rankingCount": 12,
@@ -1820,20 +1814,18 @@
18201814
"settlement-rights-delivery-baseline": {
18211815
"baselineId": "settlement-rights-delivery-baseline",
18221816
"sourceRootsPresent": true,
1823-
"sourceFileCount": 18,
1817+
"sourceFileCount": 10,
18241818
"sourceFileSample": [
1825-
"packages/generic-tools/vcs/src/prompts/CreateCommentDocCodeToolPrompt.js",
18261819
"packages/generic-tools/vcs/src/prompts/CreateCommentDocCodeToolPrompt.ts",
1827-
"packages/generic-tools/vcs/src/prompts/CreateIssueDocCodeToolPrompt.js",
18281820
"packages/generic-tools/vcs/src/prompts/CreateIssueDocCodeToolPrompt.ts",
1829-
"packages/generic-tools/vcs/src/prompts/CreateOrUpdateFileDocCodeToolPrompt.js",
18301821
"packages/generic-tools/vcs/src/prompts/CreateOrUpdateFileDocCodeToolPrompt.ts",
1831-
"packages/generic-tools/vcs/src/prompts/CreatePullRequestDocCodeToolPrompt.js",
18321822
"packages/generic-tools/vcs/src/prompts/CreatePullRequestDocCodeToolPrompt.ts",
1833-
"packages/generic-tools/vcs/src/prompts/GetFileContentDocCodeToolPrompt.js",
18341823
"packages/generic-tools/vcs/src/prompts/GetFileContentDocCodeToolPrompt.ts",
1835-
"packages/generic-tools/vcs/src/prompts/ListBranchesDocCodeToolPrompt.js",
1836-
"packages/generic-tools/vcs/src/prompts/ListBranchesDocCodeToolPrompt.ts"
1824+
"packages/generic-tools/vcs/src/prompts/ListBranchesDocCodeToolPrompt.ts",
1825+
"packages/generic-tools/vcs/src/prompts/ListRepositoriesDocCodeToolPrompt.ts",
1826+
"packages/generic-tools/vcs/src/prompts/index.ts",
1827+
"packages/pipelines/asset-pack/src/asset-pack-settlement-rights-delivery.ts",
1828+
"packages/pipelines/asset-pack/src/reading-pipeline-contract.ts"
18371829
],
18381830
"promptTemplateIdCount": 6,
18391831
"interpolatedContextKeyCount": 6,
@@ -1850,7 +1842,7 @@
18501842
"zodSchemaCount": 2,
18511843
"telemetryCount": 90,
18521844
"sourceSafetyCount": 39,
1853-
"benchmarkDefinitionCount": 2,
1845+
"benchmarkDefinitionCount": 1,
18541846
"queryPlanCount": 1,
18551847
"embeddingPolicyCount": 5,
18561848
"rankingCount": 9,

‎packages/protocol/src/canonical/v41-reading-prompt-benchmark-baselines.js‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,14 +142,17 @@ function listFiles(repoRoot, sourcePath) {
142142

143143
const files = [];
144144
const walk = (currentAbsolute, currentRelative) => {
145-
for (const entry of readdirSync(currentAbsolute, { withFileTypes: true })) {
145+
const entries = readdirSync(currentAbsolute, { withFileTypes: true });
146+
const entryNames = new Set(entries.map((entry) => entry.name));
147+
for (const entry of entries) {
146148
if (entry.name === 'dist' || entry.name === 'node_modules' || entry.name === '_legacy') continue;
147149
const nextAbsolute = path.join(currentAbsolute, entry.name);
148150
const nextRelative = path.join(currentRelative, entry.name);
149151
if (entry.isDirectory()) {
150152
walk(nextAbsolute, nextRelative);
151153
continue;
152154
}
155+
if (entry.name.endsWith('.js') && entryNames.has(`${entry.name.slice(0, -3)}.ts`)) continue;
153156
if (/\.(?:ts|tsx|js|mjs|md)$/u.test(entry.name) && !/\.d\.ts$/u.test(entry.name)) {
154157
files.push(nextRelative);
155158
}

0 commit comments

Comments
 (0)