From 82ea3a3a63663d92f5f8f37bda91158ce07db434 Mon Sep 17 00:00:00 2001 From: opencode Date: Wed, 30 Sep 2026 22:39:43 +0000 Subject: [PATCH 001/105] sync release versions for v1.18.34 --- bun.lock | 56 ++++++++++----------- packages/app/package.json | 2 +- packages/cli/package.json | 2 +- packages/codemode/package.json | 2 +- packages/console/app/package.json | 2 +- packages/console/core/package.json | 2 +- packages/console/function/package.json | 2 +- packages/console/mail/package.json | 2 +- packages/console/support/package.json | 2 +- packages/core/package.json | 2 +- packages/desktop/package.json | 2 +- packages/effect-drizzle-sqlite/package.json | 2 +- packages/effect-sqlite-node/package.json | 2 +- packages/enterprise/package.json | 2 +- packages/function/package.json | 2 +- packages/http-recorder/package.json | 2 +- packages/llm/package.json | 2 +- packages/opencode/package.json | 2 +- packages/plugin/package.json | 2 +- packages/sdk/js/package.json | 2 +- packages/server/package.json | 2 +- packages/session-ui/package.json | 2 +- packages/slack/package.json | 2 +- packages/stats/app/package.json | 2 +- packages/stats/core/package.json | 2 +- packages/stats/server/package.json | 2 +- packages/tui/package.json | 2 +- packages/ui/package.json | 2 +- packages/web/package.json | 2 +- sdks/vscode/package.json | 2 +- 30 files changed, 57 insertions(+), 57 deletions(-) diff --git a/bun.lock b/bun.lock index b415e68b6e..cb30678c4f 100644 --- a/bun.lock +++ b/bun.lock @@ -29,7 +29,7 @@ }, "packages/app": { "name": "@opencode-ai/app", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@corvu/drawer": "catalog:", "@dnd-kit/abstract": "0.5.0", @@ -96,7 +96,7 @@ }, "packages/cli": { "name": "@opencode-ai/cli", - "version": "1.18.33", + "version": "1.18.34", "bin": { "lildax": "./bin/lildax.cjs", }, @@ -144,7 +144,7 @@ }, "packages/codemode": { "name": "@opencode-ai/codemode", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "acorn": "8.15.0", "effect": "catalog:", @@ -158,7 +158,7 @@ }, "packages/console/app": { "name": "@opencode-ai/console-app", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@cloudflare/vite-plugin": "1.15.2", "@ibm/plex": "6.4.1", @@ -195,7 +195,7 @@ }, "packages/console/core": { "name": "@opencode-ai/console-core", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@aws-sdk/client-sts": "3.782.0", "@jsx-email/render": "1.1.1", @@ -222,7 +222,7 @@ }, "packages/console/function": { "name": "@opencode-ai/console-function", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@ai-sdk/anthropic": "3.0.82", "@ai-sdk/openai": "3.0.48", @@ -245,7 +245,7 @@ }, "packages/console/mail": { "name": "@opencode-ai/console-mail", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@jsx-email/all": "2.2.3", "@jsx-email/cli": "1.4.3", @@ -269,7 +269,7 @@ }, "packages/console/support": { "name": "@opencode-ai/console-support", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@cloudflare/vite-plugin": "1.15.2", "@opencode-ai/console-core": "workspace:*", @@ -289,7 +289,7 @@ }, "packages/core": { "name": "@opencode-ai/core", - "version": "1.18.33", + "version": "1.18.34", "bin": { "opencode": "./bin/opencode", }, @@ -384,7 +384,7 @@ }, "packages/desktop": { "name": "@opencode-ai/desktop", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@zip.js/zip.js": "2.7.62", "drizzle-orm": "catalog:", @@ -438,7 +438,7 @@ }, "packages/effect-drizzle-sqlite": { "name": "@opencode-ai/effect-drizzle-sqlite", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "drizzle-orm": "catalog:", "effect": "catalog:", @@ -452,7 +452,7 @@ }, "packages/effect-sqlite-node": { "name": "@opencode-ai/effect-sqlite-node", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "effect": "catalog:", }, @@ -464,7 +464,7 @@ }, "packages/enterprise": { "name": "@opencode-ai/enterprise", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@hono/standard-validator": "catalog:", "@opencode-ai/core": "workspace:*", @@ -496,7 +496,7 @@ }, "packages/function": { "name": "@opencode-ai/function", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@octokit/auth-app": "8.0.1", "@octokit/rest": "catalog:", @@ -512,7 +512,7 @@ }, "packages/http-recorder": { "name": "@opencode-ai/http-recorder", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@effect/platform-node": "4.0.0-beta.83", "@effect/platform-node-shared": "4.0.0-beta.83", @@ -543,7 +543,7 @@ }, "packages/llm": { "name": "@opencode-ai/llm", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@opencode-ai/schema": "workspace:*", "@smithy/eventstream-codec": "4.2.14", @@ -562,7 +562,7 @@ }, "packages/opencode": { "name": "opencode", - "version": "1.18.33", + "version": "1.18.34", "bin": { "opencode": "./bin/opencode", }, @@ -692,7 +692,7 @@ }, "packages/plugin": { "name": "@opencode-ai/plugin", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@ai-sdk/provider": "3.0.8", "@opencode-ai/sdk": "workspace:*", @@ -768,7 +768,7 @@ }, "packages/sdk/js": { "name": "@opencode-ai/sdk", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "cross-spawn": "catalog:", }, @@ -783,7 +783,7 @@ }, "packages/server": { "name": "@opencode-ai/server", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@opencode-ai/core": "workspace:*", "@opencode-ai/protocol": "workspace:*", @@ -798,7 +798,7 @@ }, "packages/session-ui": { "name": "@opencode-ai/session-ui", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@kobalte/core": "catalog:", "@opencode-ai/client": "file:../app/vendor/opencode-ai-client-1.17.13-v2.tgz", @@ -838,7 +838,7 @@ }, "packages/slack": { "name": "@opencode-ai/slack", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@opencode-ai/sdk": "workspace:*", "@slack/bolt": "^3.17.1", @@ -851,7 +851,7 @@ }, "packages/stats/app": { "name": "@opencode-ai/stats-app", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@ibm/plex": "6.4.1", "@kobalte/core": "catalog:", @@ -878,7 +878,7 @@ }, "packages/stats/core": { "name": "@opencode-ai/stats-core", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@aws-sdk/client-athena": "3.933.0", "@planetscale/database": "1.19.0", @@ -897,7 +897,7 @@ }, "packages/stats/server": { "name": "@opencode-ai/stats-server", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@aws-sdk/client-firehose": "3.933.0", "@effect/platform-node": "catalog:", @@ -939,7 +939,7 @@ }, "packages/tui": { "name": "@opencode-ai/tui", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@opencode-ai/core": "workspace:*", "@opencode-ai/plugin": "workspace:*", @@ -965,7 +965,7 @@ }, "packages/ui": { "name": "@opencode-ai/ui", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@kobalte/core": "catalog:", "@pierre/diffs": "catalog:", @@ -1016,7 +1016,7 @@ }, "packages/web": { "name": "@opencode-ai/web", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@astrojs/cloudflare": "12.6.3", "@astrojs/markdown-remark": "6.3.1", diff --git a/packages/app/package.json b/packages/app/package.json index 109f928dc8..811e7ede3f 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -1,6 +1,6 @@ { "name": "@opencode-ai/app", - "version": "1.18.33", + "version": "1.18.34", "description": "", "type": "module", "exports": { diff --git a/packages/cli/package.json b/packages/cli/package.json index ad2d73cc8e..ce7e9be937 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@opencode-ai/cli", - "version": "1.18.33", + "version": "1.18.34", "type": "module", "license": "MIT", "bin": { diff --git a/packages/codemode/package.json b/packages/codemode/package.json index d857d1ee90..cf9dc5a7bf 100644 --- a/packages/codemode/package.json +++ b/packages/codemode/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@opencode-ai/codemode", - "version": "1.18.33", + "version": "1.18.34", "description": "Effect-native confined code execution over schema-described tools", "private": true, "type": "module", diff --git a/packages/console/app/package.json b/packages/console/app/package.json index 2feae199ed..43dccd23a6 100644 --- a/packages/console/app/package.json +++ b/packages/console/app/package.json @@ -1,6 +1,6 @@ { "name": "@opencode-ai/console-app", - "version": "1.18.33", + "version": "1.18.34", "type": "module", "license": "MIT", "scripts": { diff --git a/packages/console/core/package.json b/packages/console/core/package.json index 4486987443..6875a26a99 100644 --- a/packages/console/core/package.json +++ b/packages/console/core/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@opencode-ai/console-core", - "version": "1.18.33", + "version": "1.18.34", "private": true, "type": "module", "license": "MIT", diff --git a/packages/console/function/package.json b/packages/console/function/package.json index ab3528b8cf..2472fee555 100644 --- a/packages/console/function/package.json +++ b/packages/console/function/package.json @@ -1,6 +1,6 @@ { "name": "@opencode-ai/console-function", - "version": "1.18.33", + "version": "1.18.34", "$schema": "https://json.schemastore.org/package.json", "private": true, "type": "module", diff --git a/packages/console/mail/package.json b/packages/console/mail/package.json index efed48ac06..2bde5f86e3 100644 --- a/packages/console/mail/package.json +++ b/packages/console/mail/package.json @@ -1,6 +1,6 @@ { "name": "@opencode-ai/console-mail", - "version": "1.18.33", + "version": "1.18.34", "dependencies": { "@jsx-email/all": "2.2.3", "@jsx-email/cli": "1.4.3", diff --git a/packages/console/support/package.json b/packages/console/support/package.json index 8c63242b52..84432143bd 100644 --- a/packages/console/support/package.json +++ b/packages/console/support/package.json @@ -1,6 +1,6 @@ { "name": "@opencode-ai/console-support", - "version": "1.18.33", + "version": "1.18.34", "type": "module", "license": "MIT", "scripts": { diff --git a/packages/core/package.json b/packages/core/package.json index 5488a1dc38..6c00d3ad6f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "$schema": "https://json.schemastore.org/package.json", - "version": "1.18.33", + "version": "1.18.34", "name": "@opencode-ai/core", "type": "module", "license": "MIT", diff --git a/packages/desktop/package.json b/packages/desktop/package.json index 4ef186353e..ad83e4e8e1 100644 --- a/packages/desktop/package.json +++ b/packages/desktop/package.json @@ -1,7 +1,7 @@ { "name": "@opencode-ai/desktop", "private": true, - "version": "1.18.33", + "version": "1.18.34", "type": "module", "license": "MIT", "homepage": "https://opencode.ai", diff --git a/packages/effect-drizzle-sqlite/package.json b/packages/effect-drizzle-sqlite/package.json index 703c40ecfb..e0b611e0c4 100644 --- a/packages/effect-drizzle-sqlite/package.json +++ b/packages/effect-drizzle-sqlite/package.json @@ -1,6 +1,6 @@ { "$schema": "https://json.schemastore.org/package.json", - "version": "1.18.33", + "version": "1.18.34", "name": "@opencode-ai/effect-drizzle-sqlite", "type": "module", "license": "MIT", diff --git a/packages/effect-sqlite-node/package.json b/packages/effect-sqlite-node/package.json index 328f048574..a0a8709b7b 100644 --- a/packages/effect-sqlite-node/package.json +++ b/packages/effect-sqlite-node/package.json @@ -1,6 +1,6 @@ { "$schema": "https://json.schemastore.org/package.json", - "version": "1.18.33", + "version": "1.18.34", "name": "@opencode-ai/effect-sqlite-node", "type": "module", "license": "MIT", diff --git a/packages/enterprise/package.json b/packages/enterprise/package.json index 31d405c805..adca0844c0 100644 --- a/packages/enterprise/package.json +++ b/packages/enterprise/package.json @@ -1,6 +1,6 @@ { "name": "@opencode-ai/enterprise", - "version": "1.18.33", + "version": "1.18.34", "private": true, "type": "module", "license": "MIT", diff --git a/packages/function/package.json b/packages/function/package.json index 5a33b79b82..dd58cbc6ec 100644 --- a/packages/function/package.json +++ b/packages/function/package.json @@ -1,6 +1,6 @@ { "name": "@opencode-ai/function", - "version": "1.18.33", + "version": "1.18.34", "$schema": "https://json.schemastore.org/package.json", "private": true, "type": "module", diff --git a/packages/http-recorder/package.json b/packages/http-recorder/package.json index c1970380d0..3ce2feefbc 100644 --- a/packages/http-recorder/package.json +++ b/packages/http-recorder/package.json @@ -1,6 +1,6 @@ { "$schema": "https://json.schemastore.org/package.json", - "version": "1.18.33", + "version": "1.18.34", "name": "@opencode-ai/http-recorder", "description": "Record and replay Effect HTTP client traffic with deterministic cassettes", "type": "module", diff --git a/packages/llm/package.json b/packages/llm/package.json index 8b0100150e..98dec95565 100644 --- a/packages/llm/package.json +++ b/packages/llm/package.json @@ -1,6 +1,6 @@ { "$schema": "https://json.schemastore.org/package.json", - "version": "1.18.33", + "version": "1.18.34", "name": "@opencode-ai/llm", "type": "module", "license": "MIT", diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 6926ddfa3e..b4e20c14e0 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -1,6 +1,6 @@ { "$schema": "https://json.schemastore.org/package.json", - "version": "1.18.33", + "version": "1.18.34", "name": "opencode", "type": "module", "license": "MIT", diff --git a/packages/plugin/package.json b/packages/plugin/package.json index dd39542378..e9a654d58e 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@opencode-ai/plugin", - "version": "1.18.33", + "version": "1.18.34", "type": "module", "license": "MIT", "scripts": { diff --git a/packages/sdk/js/package.json b/packages/sdk/js/package.json index ce29f29ba0..bdbdf4ec8f 100644 --- a/packages/sdk/js/package.json +++ b/packages/sdk/js/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@opencode-ai/sdk", - "version": "1.18.33", + "version": "1.18.34", "type": "module", "license": "MIT", "scripts": { diff --git a/packages/server/package.json b/packages/server/package.json index b31cb1d465..ce326a94ea 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@opencode-ai/server", - "version": "1.18.33", + "version": "1.18.34", "private": true, "type": "module", "license": "MIT", diff --git a/packages/session-ui/package.json b/packages/session-ui/package.json index 1761316a90..bed9794aa3 100644 --- a/packages/session-ui/package.json +++ b/packages/session-ui/package.json @@ -1,6 +1,6 @@ { "name": "@opencode-ai/session-ui", - "version": "1.18.33", + "version": "1.18.34", "private": true, "type": "module", "license": "MIT", diff --git a/packages/slack/package.json b/packages/slack/package.json index 0b29d419e9..61b6eb4fc5 100644 --- a/packages/slack/package.json +++ b/packages/slack/package.json @@ -1,6 +1,6 @@ { "name": "@opencode-ai/slack", - "version": "1.18.33", + "version": "1.18.34", "type": "module", "license": "MIT", "scripts": { diff --git a/packages/stats/app/package.json b/packages/stats/app/package.json index 48a6af3a10..b54591d0ed 100644 --- a/packages/stats/app/package.json +++ b/packages/stats/app/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@opencode-ai/stats-app", - "version": "1.18.33", + "version": "1.18.34", "private": true, "type": "module", "license": "MIT", diff --git a/packages/stats/core/package.json b/packages/stats/core/package.json index 04d3e98840..d938182cb4 100644 --- a/packages/stats/core/package.json +++ b/packages/stats/core/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@opencode-ai/stats-core", - "version": "1.18.33", + "version": "1.18.34", "private": true, "type": "module", "license": "MIT", diff --git a/packages/stats/server/package.json b/packages/stats/server/package.json index 6060cf665b..9bae345f81 100644 --- a/packages/stats/server/package.json +++ b/packages/stats/server/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@opencode-ai/stats-server", - "version": "1.18.33", + "version": "1.18.34", "private": true, "type": "module", "license": "MIT", diff --git a/packages/tui/package.json b/packages/tui/package.json index cda41f25c1..5e57c37ac5 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@opencode-ai/tui", - "version": "1.18.33", + "version": "1.18.34", "private": true, "type": "module", "license": "MIT", diff --git a/packages/ui/package.json b/packages/ui/package.json index df79f8768f..fad8b2e9b5 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -1,6 +1,6 @@ { "name": "@opencode-ai/ui", - "version": "1.18.33", + "version": "1.18.34", "type": "module", "license": "MIT", "repository": { diff --git a/packages/web/package.json b/packages/web/package.json index dfd7eb1ac6..65e6567463 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -2,7 +2,7 @@ "name": "@opencode-ai/web", "type": "module", "license": "MIT", - "version": "1.18.33", + "version": "1.18.34", "scripts": { "dev": "astro dev", "dev:remote": "VITE_API_URL=https://api.opencode.ai astro dev", diff --git a/sdks/vscode/package.json b/sdks/vscode/package.json index d865ca8c83..9b12fcea81 100644 --- a/sdks/vscode/package.json +++ b/sdks/vscode/package.json @@ -2,7 +2,7 @@ "name": "opencode", "displayName": "opencode", "description": "opencode for VS Code", - "version": "1.18.33", + "version": "1.18.34", "publisher": "sst-dev", "repository": { "type": "git", From 28e13d9fd01cde401f8891ee1cc85fa7e3715b24 Mon Sep 17 00:00:00 2001 From: "opencode-agent[bot]" <219766164+opencode-agent[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:39:52 -0500 Subject: [PATCH 002/105] fix(ci): extend issue and PR compliance grace to 72 hours (#52415) Co-authored-by: rekram1-node --- .github/workflows/compliance-close.yml | 14 +++++++------- .github/workflows/duplicate-issues.yml | 4 ++-- .github/workflows/pr-standards.yml | 2 +- CONTRIBUTING.md | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/compliance-close.yml b/.github/workflows/compliance-close.yml index a83824e5cf..cc09925c5b 100644 --- a/.github/workflows/compliance-close.yml +++ b/.github/workflows/compliance-close.yml @@ -15,7 +15,7 @@ jobs: close-non-compliant: runs-on: ubuntu-latest steps: - - name: Close non-compliant issues and PRs after 2 hours + - name: Close non-compliant issues and PRs after 72 hours uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: script: | @@ -33,7 +33,7 @@ jobs: } const now = Date.now(); - const twoHours = 2 * 60 * 60 * 1000; + const seventyTwoHours = 72 * 60 * 60 * 1000; const orgMemberAssociations = new Set(['OWNER', 'MEMBER']); const agentLogin = 'opencode-agent[bot]'; const { data: file } = await github.rest.repos.getContent({ @@ -87,14 +87,14 @@ jobs: if (!complianceComment) continue; const commentAge = now - new Date(complianceComment.created_at).getTime(); - if (commentAge < twoHours) { - core.info(`${kind} #${item.number} still within 2-hour window (${Math.round(commentAge / 60000)}m elapsed)`); + if (commentAge < seventyTwoHours) { + core.info(`${kind} #${item.number} still within 72-hour window (${Math.round(commentAge / 60000)}m elapsed)`); continue; } const closeMessage = isPR - ? 'This pull request has been automatically closed because it was not updated to meet our [contributing guidelines](../blob/dev/CONTRIBUTING.md) within the 2-hour window.\n\nFeel free to open a new pull request that follows our guidelines.' - : 'This issue has been automatically closed because it was not updated to meet our [contributing guidelines](../blob/dev/CONTRIBUTING.md) within the 2-hour window.\n\nFeel free to open a new issue that follows our issue templates.'; + ? 'This pull request has been automatically closed because it was not updated to meet our [contributing guidelines](../blob/dev/CONTRIBUTING.md) within the 72-hour window.\n\nFeel free to open a new pull request that follows our guidelines.' + : 'This issue has been automatically closed because it was not updated to meet our [contributing guidelines](../blob/dev/CONTRIBUTING.md) within the 72-hour window.\n\nFeel free to open a new issue that follows our issue templates.'; await github.rest.issues.createComment({ owner: context.repo.owner, @@ -129,5 +129,5 @@ jobs: }); } - core.info(`Closed non-compliant ${kind} #${item.number} after 2-hour window`); + core.info(`Closed non-compliant ${kind} #${item.number} after 72-hour window`); } diff --git a/.github/workflows/duplicate-issues.yml b/.github/workflows/duplicate-issues.yml index 4254cca644..be193dc7fc 100644 --- a/.github/workflows/duplicate-issues.yml +++ b/.github/workflows/duplicate-issues.yml @@ -107,7 +107,7 @@ jobs: If the issue is NOT compliant and the author association is not OWNER or MEMBER, start the comment with: - Then explain what needs to be fixed and that they have 2 hours to edit the issue before it is automatically closed. Also add the label needs:compliance to the issue using: gh issue edit ${{ github.event.issue.number }} --add-label needs:compliance + Then explain what needs to be fixed and that they have 72 hours to edit the issue before it is automatically closed. Also add the label needs:compliance to the issue using: gh issue edit ${{ github.event.issue.number }} --add-label needs:compliance If duplicates were found, include a section about potential duplicates with links. @@ -124,7 +124,7 @@ jobs: **What needs to be fixed:** - [specific reasons] - Please edit this issue to address the above within **2 hours**, or it will be automatically closed. + Please edit this issue to address the above within **72 hours**, or it will be automatically closed. [If duplicates found, add:] --- diff --git a/.github/workflows/pr-standards.yml b/.github/workflows/pr-standards.yml index 06838089d3..2984344e02 100644 --- a/.github/workflows/pr-standards.yml +++ b/.github/workflows/pr-standards.yml @@ -298,7 +298,7 @@ jobs: **What needs to be fixed:** ${issues.map(i => `- ${i}`).join('\n')} - Please edit this PR description to address the above within **2 hours**, or it will be automatically closed. + Please edit this PR description to address the above within **72 hours**, or it will be automatically closed. If you believe this was flagged incorrectly, please let a maintainer know.`; diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a9c545efb8..a53e2dd0b2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -260,7 +260,7 @@ All issues **must** use one of our issue templates: - **Feature request** — for suggesting enhancements (requires verification checkbox and description) - **Question** — for asking questions (requires the question) -Blank issues are not allowed. When a new issue is opened, an automated check verifies that it follows a template and meets our contributing guidelines. If an issue doesn't meet the requirements, you'll receive a comment explaining what needs to be fixed and have **2 hours** to edit the issue. After that, it will be automatically closed. +Blank issues are not allowed. When a new issue is opened, an automated check verifies that it follows a template and meets our contributing guidelines. If an issue doesn't meet the requirements, you'll receive a comment explaining what needs to be fixed and have **72 hours** to edit the issue. After that, it will be automatically closed. Issues may be flagged for: From 62ac31eb7ae1824bda4aafe7765983d24962be7a Mon Sep 17 00:00:00 2001 From: vprdev Date: Thu, 1 Oct 2026 00:21:33 -0400 Subject: [PATCH 003/105] chore(triage): update inference issue owners (#52424) --- .opencode/tool/github-triage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.opencode/tool/github-triage.ts b/.opencode/tool/github-triage.ts index d610a81e49..ff4668f002 100644 --- a/.opencode/tool/github-triage.ts +++ b/.opencode/tool/github-triage.ts @@ -5,7 +5,7 @@ const TEAM = { tui: ["kommander", "simonklee"], desktop_web: ["Hona", "Brendonovich"], core: ["jlongster", "rekram1-node", "neriousy", "nexxeln", "kitlangton"], - inference: ["fwang", "MrMushrooooom", "starptech"], + inference: ["fwang", "vaprdev"], windows: ["Hona"], } as const From 0112a92c416f5ad833d96e7a8308441f0a875d94 Mon Sep 17 00:00:00 2001 From: vprdev Date: Thu, 1 Oct 2026 00:27:04 -0400 Subject: [PATCH 004/105] chore(triage): add vimtor to inference issue owners (#52425) --- .opencode/tool/github-triage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.opencode/tool/github-triage.ts b/.opencode/tool/github-triage.ts index ff4668f002..fb42dc050b 100644 --- a/.opencode/tool/github-triage.ts +++ b/.opencode/tool/github-triage.ts @@ -5,7 +5,7 @@ const TEAM = { tui: ["kommander", "simonklee"], desktop_web: ["Hona", "Brendonovich"], core: ["jlongster", "rekram1-node", "neriousy", "nexxeln", "kitlangton"], - inference: ["fwang", "vaprdev"], + inference: ["fwang", "vaprdev", "vimtor"], windows: ["Hona"], } as const From 8bb2ccf82944eaec6c40b0bcdd03886bbfe86ef2 Mon Sep 17 00:00:00 2001 From: Adam <2363879+adamdotdevin@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:12:55 -0500 Subject: [PATCH 005/105] chore(stats): retire the S3 data lake (#52514) --- bun.lock | 5 - infra/console.ts | 6 +- infra/lake.ts | 89 +++-------- infra/stats.ts | 20 +-- .../console/function/src/log-processor.ts | 134 ++-------------- packages/stats/core/package.json | 2 - packages/stats/core/src/athena.ts | 151 ------------------ packages/stats/core/src/index.ts | 1 - packages/stats/core/src/resource.d.ts | 9 -- 9 files changed, 37 insertions(+), 380 deletions(-) delete mode 100644 packages/stats/core/src/athena.ts diff --git a/bun.lock b/bun.lock index cb30678c4f..89315fb65a 100644 --- a/bun.lock +++ b/bun.lock @@ -880,7 +880,6 @@ "name": "@opencode-ai/stats-core", "version": "1.18.34", "dependencies": { - "@aws-sdk/client-athena": "3.933.0", "@planetscale/database": "1.19.0", "drizzle-orm": "catalog:", "effect": "catalog:", @@ -1268,8 +1267,6 @@ "@aws-crypto/util": ["@aws-crypto/util@5.2.0", "", { "dependencies": { "@aws-sdk/types": "^3.222.0", "@smithy/util-utf8": "^2.0.0", "tslib": "^2.6.2" } }, "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ=="], - "@aws-sdk/client-athena": ["@aws-sdk/client-athena@3.933.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.932.0", "@aws-sdk/credential-provider-node": "3.933.0", "@aws-sdk/middleware-host-header": "3.930.0", "@aws-sdk/middleware-logger": "3.930.0", "@aws-sdk/middleware-recursion-detection": "3.933.0", "@aws-sdk/middleware-user-agent": "3.932.0", "@aws-sdk/region-config-resolver": "3.930.0", "@aws-sdk/types": "3.930.0", "@aws-sdk/util-endpoints": "3.930.0", "@aws-sdk/util-user-agent-browser": "3.930.0", "@aws-sdk/util-user-agent-node": "3.932.0", "@smithy/config-resolver": "^4.4.3", "@smithy/core": "^3.18.2", "@smithy/fetch-http-handler": "^5.3.6", "@smithy/hash-node": "^4.2.5", "@smithy/invalid-dependency": "^4.2.5", "@smithy/middleware-content-length": "^4.2.5", "@smithy/middleware-endpoint": "^4.3.9", "@smithy/middleware-retry": "^4.4.9", "@smithy/middleware-serde": "^4.2.5", "@smithy/middleware-stack": "^4.2.5", "@smithy/node-config-provider": "^4.3.5", "@smithy/node-http-handler": "^4.4.5", "@smithy/protocol-http": "^5.3.5", "@smithy/smithy-client": "^4.9.5", "@smithy/types": "^4.9.0", "@smithy/url-parser": "^4.2.5", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", "@smithy/util-defaults-mode-browser": "^4.3.8", "@smithy/util-defaults-mode-node": "^4.2.11", "@smithy/util-endpoints": "^3.2.5", "@smithy/util-middleware": "^4.2.5", "@smithy/util-retry": "^4.2.5", "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "sha512-9eMUCu1Ay3C9ojo+dJcynSdpbxuwDVtZUt/Xhce+c2+mgDsmvRzjww+wfLpZwRNWxBWmeauQQAZk52tCwQgXsQ=="], - "@aws-sdk/client-cognito-identity": ["@aws-sdk/client-cognito-identity@3.1057.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.15", "@aws-sdk/credential-provider-node": "^3.972.47", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/fetch-http-handler": "^5.4.5", "@smithy/node-http-handler": "^4.7.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-5MliYkp2u0+2arTp5fZIaxl+xmm90LEKv/VeSxhfNQW4t0fvWJrNO429/jchWQenNoDRrOGE59VfbuZUfwFujg=="], "@aws-sdk/client-firehose": ["@aws-sdk/client-firehose@3.933.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.932.0", "@aws-sdk/credential-provider-node": "3.933.0", "@aws-sdk/middleware-host-header": "3.930.0", "@aws-sdk/middleware-logger": "3.930.0", "@aws-sdk/middleware-recursion-detection": "3.933.0", "@aws-sdk/middleware-user-agent": "3.932.0", "@aws-sdk/region-config-resolver": "3.930.0", "@aws-sdk/types": "3.930.0", "@aws-sdk/util-endpoints": "3.930.0", "@aws-sdk/util-user-agent-browser": "3.930.0", "@aws-sdk/util-user-agent-node": "3.932.0", "@smithy/config-resolver": "^4.4.3", "@smithy/core": "^3.18.2", "@smithy/fetch-http-handler": "^5.3.6", "@smithy/hash-node": "^4.2.5", "@smithy/invalid-dependency": "^4.2.5", "@smithy/middleware-content-length": "^4.2.5", "@smithy/middleware-endpoint": "^4.3.9", "@smithy/middleware-retry": "^4.4.9", "@smithy/middleware-serde": "^4.2.5", "@smithy/middleware-stack": "^4.2.5", "@smithy/node-config-provider": "^4.3.5", "@smithy/node-http-handler": "^4.4.5", "@smithy/protocol-http": "^5.3.5", "@smithy/smithy-client": "^4.9.5", "@smithy/types": "^4.9.0", "@smithy/url-parser": "^4.2.5", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", "@smithy/util-defaults-mode-browser": "^4.3.8", "@smithy/util-defaults-mode-node": "^4.2.11", "@smithy/util-endpoints": "^3.2.5", "@smithy/util-middleware": "^4.2.5", "@smithy/util-retry": "^4.2.5", "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "sha512-tDrtgczN2lQsflLDPYu/wdOoyCZLVYtgzmWnYzSEOBWd/cp2AbuQ7D+FemSwUTzyoMTuhhIevyEJKzqsF+QYxA=="], @@ -5762,8 +5759,6 @@ "@aws-crypto/util/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="], - "@aws-sdk/client-athena/@smithy/util-utf8": ["@smithy/util-utf8@4.2.2", "", { "dependencies": { "@smithy/util-buffer-from": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-75MeYpjdWRe8M5E3AW0O4Cx3UadweS+cwdXjwYGBW5h/gxxnbeZ877sLPX/ZJA9GVTlL/qG0dXP29JWFCD1Ayw=="], - "@aws-sdk/client-cognito-identity/@aws-sdk/core": ["@aws-sdk/core@3.974.15", "", { "dependencies": { "@aws-sdk/types": "^3.973.9", "@aws-sdk/xml-builder": "^3.972.26", "@aws/lambda-invoke-store": "^0.2.2", "@smithy/core": "^3.24.5", "@smithy/signature-v4": "^5.4.5", "@smithy/types": "^4.14.2", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-UpA0rTGW/tHGITcCqHisbuuEPraYg9GG+mWmXjY5+RxZBMLGe6aL9oe0ix50LztwAcPIkGZLH0yWdMIkCM10hw=="], "@aws-sdk/client-cognito-identity/@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.47", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.41", "@aws-sdk/credential-provider-http": "^3.972.43", "@aws-sdk/credential-provider-ini": "^3.972.46", "@aws-sdk/credential-provider-process": "^3.972.41", "@aws-sdk/credential-provider-sso": "^3.972.45", "@aws-sdk/credential-provider-web-identity": "^3.972.45", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/credential-provider-imds": "^4.3.6", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-HrId+C0DWA5qDIyLG64/kjUB2RNtPypxmABnIctK+TA1P1kHlOYoE/Wf5T5tKOMKgb08P7k/zNyhvfJ3lh5Oag=="], diff --git a/infra/console.ts b/infra/console.ts index 630bb56fec..77d6fd0fc1 100644 --- a/infra/console.ts +++ b/infra/console.ts @@ -1,9 +1,7 @@ -import { deployAws, domain } from "./stage" +import { domain } from "./stage" import { EMAILOCTOPUS_API_KEY } from "./app" import { SECRET } from "./secret" -const lake = deployAws ? await import("./lake") : undefined - //////////////// // DATABASE //////////////// @@ -253,7 +251,7 @@ const SALESFORCE_INSTANCE_URL = new sst.Secret("SALESFORCE_INSTANCE_URL") const logProcessor = new sst.cloudflare.Worker("LogProcessor", { handler: "packages/console/function/src/log-processor.ts", - link: [SECRET.HoneycombApiKey, ...(lake?.lakeIngest ? [lake.lakeIngest] : [])], + link: [SECRET.HoneycombApiKey], }) new sst.cloudflare.x.SolidStart("Console", { diff --git a/infra/lake.ts b/infra/lake.ts index d4e46168e3..a6125a1ee1 100644 --- a/infra/lake.ts +++ b/infra/lake.ts @@ -1,5 +1,9 @@ import { domain } from "./stage" +// Retired in favor of the R2 lake and pending removal. Pulumi reads forceDestroy +// and retainOnDelete from state at delete time, so these settings must deploy to +// every stage before the resources are removed from code. + const current = aws.getCallerIdentityOutput({}) const partition = aws.getPartitionOutput({}) const region = aws.getRegionOutput({}) @@ -15,7 +19,7 @@ const s3TablesBucketWildcardArn = $interpolate`arn:${partition.partition}:s3tabl export const tableBucket = new aws.s3tables.TableBucket("LakeTableBucket", { name: tableBucketName, - forceDestroy: $app.stage !== "production", + forceDestroy: true, }) const s3TablesCatalog = new aws.cloudcontrol.Resource( @@ -51,19 +55,27 @@ const s3TablesCatalog = new aws.cloudcontrol.Resource( { dependsOn: [tableBucket] }, ) -const athenaResultsBucket = new aws.s3.Bucket("LakeAthenaResults", { - bucket: `opencode-${$app.stage}-lake-athena-results`, - forceDestroy: $app.stage !== "production", -}) +const athenaResultsBucket = new aws.s3.Bucket( + "LakeAthenaResults", + { + bucket: `opencode-${$app.stage}-lake-athena-results`, + forceDestroy: true, + }, + { retainOnDelete: false }, +) -const firehoseErrorBucket = new aws.s3.Bucket("LakeFirehoseErrors", { - bucket: `opencode-${$app.stage}-lake-firehose-errors`, - forceDestroy: $app.stage !== "production", -}) +const firehoseErrorBucket = new aws.s3.Bucket( + "LakeFirehoseErrors", + { + bucket: `opencode-${$app.stage}-lake-firehose-errors`, + forceDestroy: true, + }, + { retainOnDelete: false }, +) -const athenaWorkgroup = new aws.athena.Workgroup("LakeAthenaWorkgroup", { +new aws.athena.Workgroup("LakeAthenaWorkgroup", { name: `opencode-${$app.stage}-lake-workgroup`, - forceDestroy: $app.stage !== "production", + forceDestroy: true, configuration: { enforceWorkgroupConfiguration: true, publishCloudwatchMetricsEnabled: true, @@ -197,9 +209,6 @@ const firehose = new aws.kinesis.FirehoseDeliveryStream( export const lakeVpc = new sst.aws.Vpc("LakeVpc") export const lakeCluster = new sst.aws.Cluster("LakeCluster", { vpc: lakeVpc }) -export const lakeRegion = region.region -export const lakeCatalog = $interpolate`${glueCatalogName}/${tableBucket.name}` -export const lakeAthenaWorkgroup = athenaWorkgroup const ingestSecret = new random.RandomPassword("LakeIngestSecret", { length: 32 }) export const ingestSecretSsm = new aws.ssm.Parameter("LakeIngestSecretSsm", { @@ -277,55 +286,3 @@ export const lakeIngest = new sst.Linkable("LakeIngest", { secret: ingestSecret.result, }, }) - -export const lakeQueryPermissions = [ - { - actions: ["athena:StartQueryExecution", "athena:GetQueryExecution", "athena:GetQueryResults"], - resources: [athenaWorkgroup.arn], - }, - { - actions: [ - "glue:GetCatalog", - "glue:GetCatalogs", - "glue:GetDatabase", - "glue:GetDatabases", - "glue:GetTable", - "glue:GetTables", - "glue:GetPartitions", - ], - resources: [ - glueCatalogArn, - glueS3TablesCatalogArn, - $interpolate`${glueS3TablesCatalogArn}/*`, - glueS3TablesDatabaseWildcardArn, - glueS3TablesTableWildcardArn, - $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/*`, - $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/*/*`, - $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/*`, - ], - }, - { - actions: ["s3:GetBucketLocation", "s3:ListBucket"], - resources: [athenaResultsBucket.arn], - }, - { - actions: ["s3:GetObject", "s3:PutObject", "s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads"], - resources: [$interpolate`${athenaResultsBucket.arn}/*`], - }, - { - actions: [ - "s3tables:GetTableBucket", - "s3tables:GetNamespace", - "s3tables:GetTable", - "s3tables:GetTableData", - "s3tables:GetTableMetadataLocation", - "s3tables:ListNamespaces", - "s3tables:ListTables", - ], - resources: ["*"], - }, - { - actions: ["lakeformation:GetDataAccess"], - resources: ["*"], - }, -] diff --git a/infra/stats.ts b/infra/stats.ts index 29c9537daf..a867aa695a 100644 --- a/infra/stats.ts +++ b/infra/stats.ts @@ -1,4 +1,4 @@ -import { lakeAthenaWorkgroup, lakeCatalog, lakeCluster, lakeQueryPermissions, lakeRegion, tableBucket } from "./lake" +import { lakeCluster, tableBucket } from "./lake" import { EMAILOCTOPUS_API_KEY } from "./app" import { domain } from "./stage" @@ -11,7 +11,7 @@ const inferenceNamespace = new aws.s3tables.Namespace("LakeInferenceNamespace", tableBucketArn: tableBucket.arn, }) -const inferenceEventTable = new aws.s3tables.Table( +new aws.s3tables.Table( "LakeInferenceEventTable", { name: "event", @@ -89,17 +89,6 @@ const inferenceEventTable = new aws.s3tables.Table( { deleteBeforeReplace: $app.stage !== "production", ignoreChanges: ["metadata"] }, ) -export const inferenceEvent = new sst.Linkable("InferenceEvent", { - properties: { - region: lakeRegion, - catalog: lakeCatalog, - database: inferenceNamespace.namespace, - table: inferenceEventTable.name, - tableBucket: tableBucket.name, - workgroup: lakeAthenaWorkgroup.name, - }, -}) - //////////////// // DATABASE //////////////// @@ -203,10 +192,7 @@ export const statSync = new sst.aws.Service("StatsSyncService", { dockerfile: "packages/stats/server/Dockerfile", }, command: ["bun", "src/stat-sync.ts"], - // Keep the legacy Athena link and IAM permissions during the first R2-backed - // release so reverting the application code remains a one-deploy rollback. - link: [database, inferenceEvent, r2Sql, r2SqlAuthToken, statsSyncConfig], - permissions: lakeQueryPermissions, + link: [database, r2Sql, r2SqlAuthToken, statsSyncConfig], scaling: { min: 1, max: 1, diff --git a/packages/console/function/src/log-processor.ts b/packages/console/function/src/log-processor.ts index 75daba2b50..71f57c37b3 100644 --- a/packages/console/function/src/log-processor.ts +++ b/packages/console/function/src/log-processor.ts @@ -37,6 +37,7 @@ export default { "ip.prefix": ipPrefix(ip), } const time = new Date(event.eventTimestamp ?? Date.now()).toISOString() + // This also merges each _metric log into data, so keep it while the Honeycomb export below is disabled. const events = [ ...event.logs.flatMap((log) => log.message.flatMap((message: string) => { @@ -53,107 +54,20 @@ export default { ] console.log(JSON.stringify(data, null, 2)) - const lakeIngest = getLakeIngest() - const [lake] = await Promise.all([ - // fetch("https://api.honeycomb.io/1/batch/zen", { - // method: "POST", - // headers: { - // "Content-Type": "application/json", - // "X-Honeycomb-Team": Resource.HONEYCOMB_API_KEY.value, - // }, - // body: JSON.stringify(events), - // }), - ...(lakeIngest - ? [ - fetch(lakeIngest.url, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${lakeIngest.secret}`, - }, - body: JSON.stringify({ events: events.map((event) => toLakeEvent(event.time, event.data)) }), - }), - ] - : []), - ]) + // const honeycomb = await fetch("https://api.honeycomb.io/1/batch/zen", { + // method: "POST", + // headers: { + // "Content-Type": "application/json", + // "X-Honeycomb-Team": Resource.HONEYCOMB_API_KEY.value, + // }, + // body: JSON.stringify(events), + // }) // console.log(honeycomb.status) // console.log(await honeycomb.text()) - if (lake) { - console.log(lake.status) - console.log(await lake.text()) - } } }, } -function getLakeIngest(): { url: string; secret: string } | undefined { - try { - return Resource.LakeIngest - } catch { - return undefined - } -} - -function toLakeEvent(time: string, data: Record) { - return { - _datalake_key: "inference.event", - event_timestamp: time, - event_date: time.slice(0, 10), - event_type: string(data, "event_type"), - dataset: "zen", - cf_continent: string(data, "cf.continent"), - cf_country: string(data, "cf.country"), - cf_city: string(data, "cf.city"), - cf_region: string(data, "cf.region"), - cf_latitude: number(data, "cf.latitude"), - cf_longitude: number(data, "cf.longitude"), - cf_timezone: string(data, "cf.timezone"), - duration: number(data, "duration"), - request_length: integer(data, "request_length"), - status: integer(data, "status"), - ip: string(data, "ip"), - ip_prefix: string(data, "ip.prefix"), - is_stream: boolean(data, "is_stream"), - session: string(data, "session"), - request: string(data, "request"), - client: string(data, "client"), - user_agent: string(data, "user_agent"), - model: string(data, "model"), - model_tier: string(data, "model.tier"), - model_variant: string(data, "model.variant"), - source: string(data, "source"), - provider: string(data, "provider"), - provider_model: string(data, "provider.model"), - llm_error_code: integer(data, "llm.error.code"), - llm_error_message: string(data, "llm.error.message"), - error_response: string(data, "error.response"), - error_type: string(data, "error.type"), - error_message: string(data, "error.message"), - error_cause: string(data, "error.cause"), - error_cause2: string(data, "error.cause2"), - api_key: string(data, "api_key"), - workspace: string(data, "workspace"), - user_id: string(data, "user_id"), - is_subscription: boolean(data, "isSubscription"), // removed - subscription: string(data, "subscription"), - response_length: integer(data, "response_length"), - time_to_first_byte: integer(data, "time_to_first_byte"), - timestamp_first_byte: integer(data, "timestamp.first_byte"), - timestamp_last_byte: integer(data, "timestamp.last_byte"), - tokens_input: integer(data, "tokens.input"), - tokens_output: integer(data, "tokens.output"), - tokens_reasoning: integer(data, "tokens.reasoning"), - tokens_cache_read: integer(data, "tokens.cache_read"), - tokens_cache_write_5m: integer(data, "tokens.cache_write_5m"), - tokens_cache_write_1h: integer(data, "tokens.cache_write_1h"), - cost_input_microcents: integer(data, "cost.input.microcents"), - cost_output_microcents: integer(data, "cost.output.microcents"), - cost_cache_read_microcents: integer(data, "cost.cache_read.microcents"), - cost_cache_write_microcents: integer(data, "cost.cache_write.microcents"), - cost_total_microcents: integer(data, "cost.total.microcents"), - } -} - // Returns a stable lookup key for an IP address. // IPv4: full address as /32 (e.g. "203.0.113.45/32"). // IPv6: the /64 network prefix (e.g. "2001:db8:abcd:1234::/64"). ISPs commonly @@ -179,33 +93,3 @@ function ipPrefix(ip: string | undefined) { .join(":") return `${prefix}::/64` } - -function string(data: Record, key: string) { - const value = data[key] - if (typeof value === "string") return value - if (typeof value === "number" || typeof value === "boolean") return String(value) - return undefined -} - -function boolean(data: Record, key: string) { - const value = data[key] - if (typeof value === "boolean") return value - if (typeof value === "string") return value === "true" ? true : value === "false" ? false : undefined - return undefined -} - -function integer(data: Record, key: string) { - const value = number(data, key) - if (value === undefined) return undefined - return Math.round(value) -} - -function number(data: Record, key: string) { - const value = data[key] - if (typeof value === "number") return Number.isFinite(value) ? value : undefined - if (typeof value === "string") { - const parsed = Number(value) - return Number.isFinite(parsed) ? parsed : undefined - } - return undefined -} diff --git a/packages/stats/core/package.json b/packages/stats/core/package.json index d938182cb4..aefcf4d8cf 100644 --- a/packages/stats/core/package.json +++ b/packages/stats/core/package.json @@ -7,7 +7,6 @@ "license": "MIT", "exports": { ".": "./src/index.ts", - "./athena": "./src/athena.ts", "./config": "./src/config.ts", "./database": "./src/database.ts", "./database/*": "./src/database/*.ts", @@ -27,7 +26,6 @@ "typecheck": "tsgo --noEmit" }, "dependencies": { - "@aws-sdk/client-athena": "3.933.0", "@planetscale/database": "1.19.0", "drizzle-orm": "catalog:", "effect": "catalog:", diff --git a/packages/stats/core/src/athena.ts b/packages/stats/core/src/athena.ts deleted file mode 100644 index 23aba70b3c..0000000000 --- a/packages/stats/core/src/athena.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { - AthenaClient as AwsAthenaClient, - GetQueryExecutionCommand, - GetQueryResultsCommand, - StartQueryExecutionCommand, - type Row, -} from "@aws-sdk/client-athena" -import { Effect, Layer } from "effect" -import * as Context from "effect/Context" -import { Resource } from "sst/resource" - -const ATHENA_MAX_POLL_ATTEMPTS = 900 -const ATHENA_PAGE_SIZE = 1000 - -export type AthenaData = Record - -export class AthenaQueryError extends Error { - readonly _tag = "AthenaQueryError" - readonly queryExecutionId?: string - - constructor(input: { message: string; queryExecutionId?: string; cause?: unknown }) { - super(input.message, { cause: input.cause }) - this.name = "AthenaQueryError" - this.queryExecutionId = input.queryExecutionId - } -} - -export class AthenaQueryTimeoutError extends Error { - readonly _tag = "AthenaQueryTimeoutError" - readonly queryExecutionId: string - - constructor(input: { message: string; queryExecutionId: string }) { - super(input.message) - this.name = "AthenaQueryTimeoutError" - this.queryExecutionId = input.queryExecutionId - } -} - -export declare namespace Athena { - export interface Service { - readonly query: (query: string) => Effect.Effect - } -} - -export class Athena extends Context.Service()("@opencode/stats/Athena") { - static readonly layer: Layer.Layer = Layer.effect( - Athena, - Effect.sync(() => { - const client = new AwsAthenaClient({ region: Resource.InferenceEvent.region }) - - const query = Effect.fn("Athena.query")(function* (query: string) { - const started = yield* Effect.tryPromise({ - try: () => - client.send( - new StartQueryExecutionCommand({ - QueryString: query, - WorkGroup: Resource.InferenceEvent.workgroup, - QueryExecutionContext: { - Catalog: Resource.InferenceEvent.catalog, - Database: Resource.InferenceEvent.database, - }, - }), - ), - catch: (cause) => new AthenaQueryError({ message: "Failed to start Athena stats query", cause }), - }) - const queryExecutionId = started.QueryExecutionId - if (!queryExecutionId) - return yield* Effect.fail(new AthenaQueryError({ message: "Athena did not return a query execution id" })) - - yield* poll(client, queryExecutionId) - return yield* results(client, queryExecutionId) - }) - - return Athena.of({ query }) - }), - ) -} - -const poll: ( - client: AwsAthenaClient, - queryExecutionId: string, - attempt?: number, -) => Effect.Effect = Effect.fn("Athena.poll")(function* ( - client: AwsAthenaClient, - queryExecutionId: string, - attempt = 0, -) { - if (attempt > 0) yield* Effect.sleep("2 seconds") - - const result = yield* Effect.tryPromise({ - try: () => client.send(new GetQueryExecutionCommand({ QueryExecutionId: queryExecutionId })), - catch: (cause) => new AthenaQueryError({ message: "Failed to poll Athena stats query", queryExecutionId, cause }), - }) - const status = result.QueryExecution?.Status - - if (status?.State === "SUCCEEDED") return - if (status?.State === "FAILED" || status?.State === "CANCELLED") - return yield* Effect.fail( - new AthenaQueryError({ - message: `Athena stats query ${status.State.toLowerCase()}: ${status.StateChangeReason ?? "unknown reason"}`, - queryExecutionId, - }), - ) - - if (attempt >= ATHENA_MAX_POLL_ATTEMPTS - 1) - return yield* Effect.fail( - new AthenaQueryTimeoutError({ - message: `Athena stats query ${queryExecutionId} did not complete`, - queryExecutionId, - }), - ) - - return yield* poll(client, queryExecutionId, attempt + 1) -}) - -const results: (client: AwsAthenaClient, queryExecutionId: string) => Effect.Effect = - Effect.fn("Athena.results")(function* (client: AwsAthenaClient, queryExecutionId: string) { - // Accumulate pages iteratively; recursive spreads copied every previously - // fetched row per page and blew up memory on large result sets. - const rows: AthenaData[] = [] - let nextToken: string | undefined - while (true) { - const result = yield* Effect.tryPromise({ - try: () => - client.send( - new GetQueryResultsCommand({ - QueryExecutionId: queryExecutionId, - NextToken: nextToken, - MaxResults: ATHENA_PAGE_SIZE, - }), - ), - catch: (cause) => - new AthenaQueryError({ message: "Failed to read Athena stats results", queryExecutionId, cause }), - }) - const columns = result.ResultSet?.ResultSetMetadata?.ColumnInfo?.map((item) => item.Name ?? "") ?? [] - // The first page starts with the header row. - for (const row of (result.ResultSet?.Rows ?? []).slice(nextToken ? 0 : 1)) rows.push(rowData(columns, row)) - if (!result.NextToken) return rows - nextToken = result.NextToken - } - }) - -function rowData(columns: string[], row: Row): AthenaData { - return Object.fromEntries( - columns.flatMap((column, index) => { - const value = row.Data?.[index]?.VarCharValue - if (!column || value === undefined) return [] - return [[column, value]] - }), - ) -} diff --git a/packages/stats/core/src/index.ts b/packages/stats/core/src/index.ts index 834625f601..9b7fcd0f69 100644 --- a/packages/stats/core/src/index.ts +++ b/packages/stats/core/src/index.ts @@ -1,4 +1,3 @@ -export * as Athena from "./athena" export * as AppConfig from "./config" export * as Database from "./database" export * as GeoStat from "./domain/geo" diff --git a/packages/stats/core/src/resource.d.ts b/packages/stats/core/src/resource.d.ts index b801777797..db2167b033 100644 --- a/packages/stats/core/src/resource.d.ts +++ b/packages/stats/core/src/resource.d.ts @@ -2,15 +2,6 @@ import "sst/resource" declare module "sst/resource" { export interface Resource { - InferenceEvent: { - catalog: string - database: string - region: string - table: string - tableBucket: string - type: "sst.sst.Linkable" - workgroup: string - } R2Sql: { accountId: string bucket: string From 63cf236140b11c15cd645e90a7ded77bdc2193b4 Mon Sep 17 00:00:00 2001 From: "opencode-agent[bot]" Date: Thu, 1 Oct 2026 15:37:36 +0000 Subject: [PATCH 006/105] chore: update nix node_modules hashes --- nix/hashes.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nix/hashes.json b/nix/hashes.json index 3672243020..b4cef486b6 100644 --- a/nix/hashes.json +++ b/nix/hashes.json @@ -1,8 +1,8 @@ { "nodeModules": { - "x86_64-linux": "sha256-RIzpvDBR6GOfU2vjrtgP3LJ/j+gLxkMnJ9VE5Y65Pfo=", - "aarch64-linux": "sha256-seE7v8AqxWCiyxNARMiLJNB1MZv15OwEf6r21k68xqg=", - "aarch64-darwin": "sha256-rbyK+3qz9bGOxg3HlGW6yh3qeazCaxFcuPfdGbXHsIY=", - "x86_64-darwin": "sha256-chSE5WtSQ+qdbj5vTvGmr4rn9gk8eVp3pVeow4gVqYI=" + "x86_64-linux": "sha256-fAqsf8Zgjz9oinwnFmlDbnT8hW61MfP1aJgXDSWFHLg=", + "aarch64-linux": "sha256-Ite4U+M3tagfb/0f33XwNdXZRpbQDFD5g1SuKQMU9wo=", + "aarch64-darwin": "sha256-53v/BWVbpOFQqP+Wc906FWOih0xKbQgiumGbqmIS1fo=", + "x86_64-darwin": "sha256-tjWqIKUiwHg4goIb/aHy3EOEvYcjjK4ft0GHPiPhVbY=" } } From aa481b8f5652f5576c55f914a64ed270e7daa7e0 Mon Sep 17 00:00:00 2001 From: Adam <2363879+adamdotdevin@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:24:41 -0500 Subject: [PATCH 007/105] fix(stats): fix data page canonicals, 404s, and sitemap coverage (#52525) --- .../src/component/model-compare-detail.tsx | 33 +++++++---- .../app/src/component/not-found-meta.tsx | 12 ++++ packages/stats/app/src/routes/[...404].tsx | 18 ++++++ .../stats/app/src/routes/[lab]/[model].tsx | 58 +++++++++++-------- packages/stats/app/src/routes/[lab]/index.tsx | 45 ++++++++------ .../compare/[firstFamily]/[secondFamily].tsx | 5 +- .../stats/app/src/routes/compare/index.tsx | 39 +++++++------ packages/stats/app/src/routes/index.tsx | 39 +++++++------ .../stats/app/src/routes/model-catalog.ts | 4 ++ packages/stats/app/src/routes/sitemap.xml.ts | 22 ++++++- 10 files changed, 180 insertions(+), 95 deletions(-) create mode 100644 packages/stats/app/src/component/not-found-meta.tsx create mode 100644 packages/stats/app/src/routes/[...404].tsx diff --git a/packages/stats/app/src/component/model-compare-detail.tsx b/packages/stats/app/src/component/model-compare-detail.tsx index e4c2ac5cbd..8d59cf5dc4 100644 --- a/packages/stats/app/src/component/model-compare-detail.tsx +++ b/packages/stats/app/src/component/model-compare-detail.tsx @@ -48,6 +48,7 @@ import { } from "../lib/comparison-pages" import { baseUrl } from "../lib/language" import { runStatsEffect } from "../stats-runtime" +import { NotFoundMeta } from "./not-found-meta" const compareHeaderLinks: readonly HeaderLink[] = [ { href: `${import.meta.env.BASE_URL}#top-models`, label: "Top Models" }, @@ -235,20 +236,28 @@ export default function ModelCompareDetailPage(props: ModelCompareDetailPageProp return (
- + {/* Server-rendered head tags are never removed, so render them once all data has loaded. */} + {title()} - 2 ? "noindex,follow" : "index,follow"} /> - - - - - - - - - - + model.catalog || model.stats)} + fallback={} + > + 2 ? "noindex,follow" : "index,follow"} /> + + + + + + + + + + +
+ + + + ) +} diff --git a/packages/stats/app/src/routes/[...404].tsx b/packages/stats/app/src/routes/[...404].tsx new file mode 100644 index 0000000000..62a03c0e1c --- /dev/null +++ b/packages/stats/app/src/routes/[...404].tsx @@ -0,0 +1,18 @@ +import { Title } from "@solidjs/meta" +import { NotFoundMeta } from "../component/not-found-meta" +import { useLanguage } from "../context/language" + +export default function NotFound() { + const language = useLanguage() + return ( +
+ Page not found + +
+ Page not found +

This data page doesn't exist.

+ Model data +
+
+ ) +} diff --git a/packages/stats/app/src/routes/[lab]/[model].tsx b/packages/stats/app/src/routes/[lab]/[model].tsx index 6753afa3cb..5cd33cafea 100644 --- a/packages/stats/app/src/routes/[lab]/[model].tsx +++ b/packages/stats/app/src/routes/[lab]/[model].tsx @@ -14,10 +14,12 @@ import { createAsync, query, useParams } from "@solidjs/router" import { createMemo, createSignal, createUniqueId, For, onMount, Show, type JSX } from "solid-js" import { getRequestEvent } from "solid-js/web" import { LocaleLinks } from "../../component/locale-links" +import { NotFoundMeta } from "../../component/not-found-meta" import { useI18n } from "../../context/i18n" import { useLanguage } from "../../context/language" import { localizedUrl } from "../../lib/language" import { + catalogModelPath, findModelCatalogEntry, formatCatalogLabName, isKnownCatalogLab, @@ -47,7 +49,7 @@ import { } from "../stats-shell" const statsUnfurlPath = "banner.png" -const glmFlashCatalogId = "zhipuai/glm-5.3-flash" +const glmFlashPath = "/data/zhipuai/glm-5-3-flash" const glmFlashModel = "glm-5.3-flash" const shortMonths = ["JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC"] as const @@ -110,10 +112,10 @@ export default function StatsModel() { const modelTitle = createMemo(() => i18n.t("model.title", { model: searchModelName() })) const modelDescription = createMemo(() => i18n.t("model.description", { model: searchModelName() })) const modelPath = createMemo(() => { - const fallback = formerName() - ? glmFlashCatalogId - : [labParam(), stats()?.slug ?? canonicalModel()].filter((part) => part.length > 0).join("/") - return `/data/${catalogEntry()?.id ?? fallback}` + const entry = catalogEntry() + if (entry) return catalogModelPath(entry) + if (formerName()) return glmFlashPath + return `/data/${providerSlug(labParam())}/${stats()?.slug ?? providerSlug(canonicalModel())}` }) const modelUrl = createMemo(() => localizedUrl(language.locale(), modelPath())) const statsUnfurlUrl = new URL(statsUnfurlPath, localizedUrl("en", "/data/")).toString() @@ -152,24 +154,32 @@ export default function StatsModel() { return (
- {modelTitle()} - - - - - - - - - - - - - - - - - + {/* Server-rendered head tags are never removed, so render them once data has loaded. */} + + {modelTitle()} + + } + > + + + + + + + + + + + + + + + + + +
({ - href: language.route(`${import.meta.env.BASE_URL}${model.id}`), + href: language.route(catalogModelPath(model)), label: model.name, value: model.id, }))} diff --git a/packages/stats/app/src/routes/[lab]/index.tsx b/packages/stats/app/src/routes/[lab]/index.tsx index f79f5c21f4..980602d826 100644 --- a/packages/stats/app/src/routes/[lab]/index.tsx +++ b/packages/stats/app/src/routes/[lab]/index.tsx @@ -12,10 +12,12 @@ import { createAsync, query, useParams } from "@solidjs/router" import { createMemo, createSignal, createUniqueId, For, onMount, Show, type JSX } from "solid-js" import { getRequestEvent } from "solid-js/web" import { LocaleLinks } from "../../component/locale-links" +import { NotFoundMeta } from "../../component/not-found-meta" import { useI18n } from "../../context/i18n" import { useLanguage } from "../../context/language" import { localizedUrl } from "../../lib/language" import { + catalogModelPath, catalogSlug, findModelCatalogLab, formatCatalogLabName, @@ -124,24 +126,29 @@ export default function StatsLab() { return (
- {labTitle()} - - - - - - - - - - - - - - - - - + {/* Server-rendered head tags are never removed, so render them once data has loaded. */} + + {labTitle()} + + }> + + + + + + + + + + + + + + + + + +
props.onTooltipChange(undefined)} diff --git a/packages/stats/app/src/routes/compare/[firstFamily]/[secondFamily].tsx b/packages/stats/app/src/routes/compare/[firstFamily]/[secondFamily].tsx index a39226721b..bdf130d843 100644 --- a/packages/stats/app/src/routes/compare/[firstFamily]/[secondFamily].tsx +++ b/packages/stats/app/src/routes/compare/[firstFamily]/[secondFamily].tsx @@ -1,7 +1,8 @@ -import { Meta, Title } from "@solidjs/meta" +import { Title } from "@solidjs/meta" import { createAsync, useParams } from "@solidjs/router" import { createMemo, Show } from "solid-js" import ModelCompareDetailPage from "../../../component/model-compare-detail" +import { NotFoundMeta } from "../../../component/not-found-meta" import { resolveComparisonFamily } from "../../../lib/comparison-pages" import { getModelCatalog } from "../../model-catalog" @@ -23,7 +24,7 @@ export default function ModelCompareFamily() { fallback={ Model comparison not found - +
Comparison not found diff --git a/packages/stats/app/src/routes/compare/index.tsx b/packages/stats/app/src/routes/compare/index.tsx index 861f694934..c0d1d47c7a 100644 --- a/packages/stats/app/src/routes/compare/index.tsx +++ b/packages/stats/app/src/routes/compare/index.tsx @@ -108,24 +108,27 @@ export default function ModelCompareIndex() { return (
- {compareTitle} - - - - - - - - - - - - - - - - - + {/* Server-rendered head tags are never removed, so render them once data has loaded. */} + + {compareTitle} + + + + + + + + + + + + + + + + + +
- {i18n.t("app.title")} - - - - - - - - - - - - - - - - - + {/* Server-rendered head tags are never removed, so render them once data has loaded. */} + + {i18n.t("app.title")} + + + + + + + + + + + + + + + + + +
diff --git a/packages/stats/app/src/routes/model-catalog.ts b/packages/stats/app/src/routes/model-catalog.ts index ae55eb55a8..868ea7fa20 100644 --- a/packages/stats/app/src/routes/model-catalog.ts +++ b/packages/stats/app/src/routes/model-catalog.ts @@ -91,6 +91,10 @@ export function findModelCatalogLab(catalog: ModelCatalog, lab: string) { return catalog.labs.find((entry) => entry.id === id) } +export function catalogModelPath(entry: Pick) { + return `/data/${entry.lab}/${entry.slug}` +} + export function formatCatalogLabName(lab: string) { const known: Record = { alibaba: "Alibaba", diff --git a/packages/stats/app/src/routes/sitemap.xml.ts b/packages/stats/app/src/routes/sitemap.xml.ts index d3d8cf64c6..2234e3756b 100644 --- a/packages/stats/app/src/routes/sitemap.xml.ts +++ b/packages/stats/app/src/routes/sitemap.xml.ts @@ -9,7 +9,7 @@ import { resolveComparisonFamily, } from "../lib/comparison-pages" import { baseUrl } from "../lib/language" -import { loadModelCatalog } from "./model-catalog" +import { catalogModelPath, findModelCatalogEntry, loadModelCatalog } from "./model-catalog" type SitemapEntry = { path: string @@ -55,7 +55,25 @@ export async function GET() { ] }), ) - const entries = uniqueSitemapEntries([{ path: "/data/compare", lastmod }, ...familyComparisons, ...modelComparisons]) + const labPages = catalog.labs.map((lab) => ({ + path: `/data/${lab.id}`, + lastmod: sitemapDate(stats?.updatedAt, ...lab.models.map((model) => model.lastUpdated ?? model.releaseDate)), + })) + // Skip entries whose URL resolves to another model, such as stats aliases. + const modelPages = [...catalog.models, ...(catalog.aliases ?? [])] + .filter((model) => findModelCatalogEntry(catalog, model.slug, model.lab)?.id === model.id) + .map((model) => ({ + path: catalogModelPath(model), + lastmod: sitemapDate(stats?.updatedAt, model.lastUpdated ?? model.releaseDate), + })) + const entries = uniqueSitemapEntries([ + { path: "/data/", lastmod: sitemapDate(stats?.updatedAt) }, + { path: "/data/compare", lastmod }, + ...labPages, + ...modelPages, + ...familyComparisons, + ...modelComparisons, + ]) return new Response(sitemapXml(entries), { headers: { From dc31828006702f1a88c0113b69afccad389832e9 Mon Sep 17 00:00:00 2001 From: Adam <2363879+adamdotdevin@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:58:14 -0500 Subject: [PATCH 008/105] chore(stats): remove s3 lake ingestion (#52572) --- bun.lock | 5 - infra/lake.ts | 215 +----------------------- packages/stats/server/Dockerfile | 4 +- packages/stats/server/package.json | 7 +- packages/stats/server/src/ingest.ts | 166 ------------------ packages/stats/server/src/resource.d.ts | 11 -- packages/stats/server/src/router.ts | 73 -------- packages/stats/server/src/server.ts | 28 --- packages/stats/server/src/shutdown.ts | 17 -- sst.config.ts | 7 - 10 files changed, 12 insertions(+), 521 deletions(-) delete mode 100644 packages/stats/server/src/ingest.ts delete mode 100644 packages/stats/server/src/resource.d.ts delete mode 100644 packages/stats/server/src/router.ts delete mode 100644 packages/stats/server/src/server.ts delete mode 100644 packages/stats/server/src/shutdown.ts diff --git a/bun.lock b/bun.lock index 89315fb65a..5ca5f3ac46 100644 --- a/bun.lock +++ b/bun.lock @@ -898,7 +898,6 @@ "name": "@opencode-ai/stats-server", "version": "1.18.34", "dependencies": { - "@aws-sdk/client-firehose": "3.933.0", "@effect/platform-node": "catalog:", "@opencode-ai/stats-core": "workspace:*", "effect": "catalog:", @@ -1269,8 +1268,6 @@ "@aws-sdk/client-cognito-identity": ["@aws-sdk/client-cognito-identity@3.1057.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.15", "@aws-sdk/credential-provider-node": "^3.972.47", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/fetch-http-handler": "^5.4.5", "@smithy/node-http-handler": "^4.7.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-5MliYkp2u0+2arTp5fZIaxl+xmm90LEKv/VeSxhfNQW4t0fvWJrNO429/jchWQenNoDRrOGE59VfbuZUfwFujg=="], - "@aws-sdk/client-firehose": ["@aws-sdk/client-firehose@3.933.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.932.0", "@aws-sdk/credential-provider-node": "3.933.0", "@aws-sdk/middleware-host-header": "3.930.0", "@aws-sdk/middleware-logger": "3.930.0", "@aws-sdk/middleware-recursion-detection": "3.933.0", "@aws-sdk/middleware-user-agent": "3.932.0", "@aws-sdk/region-config-resolver": "3.930.0", "@aws-sdk/types": "3.930.0", "@aws-sdk/util-endpoints": "3.930.0", "@aws-sdk/util-user-agent-browser": "3.930.0", "@aws-sdk/util-user-agent-node": "3.932.0", "@smithy/config-resolver": "^4.4.3", "@smithy/core": "^3.18.2", "@smithy/fetch-http-handler": "^5.3.6", "@smithy/hash-node": "^4.2.5", "@smithy/invalid-dependency": "^4.2.5", "@smithy/middleware-content-length": "^4.2.5", "@smithy/middleware-endpoint": "^4.3.9", "@smithy/middleware-retry": "^4.4.9", "@smithy/middleware-serde": "^4.2.5", "@smithy/middleware-stack": "^4.2.5", "@smithy/node-config-provider": "^4.3.5", "@smithy/node-http-handler": "^4.4.5", "@smithy/protocol-http": "^5.3.5", "@smithy/smithy-client": "^4.9.5", "@smithy/types": "^4.9.0", "@smithy/url-parser": "^4.2.5", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", "@smithy/util-defaults-mode-browser": "^4.3.8", "@smithy/util-defaults-mode-node": "^4.2.11", "@smithy/util-endpoints": "^3.2.5", "@smithy/util-middleware": "^4.2.5", "@smithy/util-retry": "^4.2.5", "@smithy/util-utf8": "^4.2.0", "tslib": "^2.6.2" } }, "sha512-tDrtgczN2lQsflLDPYu/wdOoyCZLVYtgzmWnYzSEOBWd/cp2AbuQ7D+FemSwUTzyoMTuhhIevyEJKzqsF+QYxA=="], - "@aws-sdk/client-lambda": ["@aws-sdk/client-lambda@3.1057.0", "", { "dependencies": { "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "^3.974.15", "@aws-sdk/credential-provider-node": "^3.972.47", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/fetch-http-handler": "^5.4.5", "@smithy/node-http-handler": "^4.7.5", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-JoaE3QNvPqOSHJtcAFfza7BRoGUNerIKlSVhsKCBsa1i54Vto1YWUS7itkUELK2rpvS8kNZMPliPxDdi/oV5dw=="], "@aws-sdk/client-s3": ["@aws-sdk/client-s3@3.933.0", "", { "dependencies": { "@aws-crypto/sha1-browser": "5.2.0", "@aws-crypto/sha256-browser": "5.2.0", "@aws-crypto/sha256-js": "5.2.0", "@aws-sdk/core": "3.932.0", "@aws-sdk/credential-provider-node": "3.933.0", "@aws-sdk/middleware-bucket-endpoint": "3.930.0", "@aws-sdk/middleware-expect-continue": "3.930.0", "@aws-sdk/middleware-flexible-checksums": "3.932.0", "@aws-sdk/middleware-host-header": "3.930.0", "@aws-sdk/middleware-location-constraint": "3.930.0", "@aws-sdk/middleware-logger": "3.930.0", "@aws-sdk/middleware-recursion-detection": "3.933.0", "@aws-sdk/middleware-sdk-s3": "3.932.0", "@aws-sdk/middleware-ssec": "3.930.0", "@aws-sdk/middleware-user-agent": "3.932.0", "@aws-sdk/region-config-resolver": "3.930.0", "@aws-sdk/signature-v4-multi-region": "3.932.0", "@aws-sdk/types": "3.930.0", "@aws-sdk/util-endpoints": "3.930.0", "@aws-sdk/util-user-agent-browser": "3.930.0", "@aws-sdk/util-user-agent-node": "3.932.0", "@smithy/config-resolver": "^4.4.3", "@smithy/core": "^3.18.2", "@smithy/eventstream-serde-browser": "^4.2.5", "@smithy/eventstream-serde-config-resolver": "^4.3.5", "@smithy/eventstream-serde-node": "^4.2.5", "@smithy/fetch-http-handler": "^5.3.6", "@smithy/hash-blob-browser": "^4.2.6", "@smithy/hash-node": "^4.2.5", "@smithy/hash-stream-node": "^4.2.5", "@smithy/invalid-dependency": "^4.2.5", "@smithy/md5-js": "^4.2.5", "@smithy/middleware-content-length": "^4.2.5", "@smithy/middleware-endpoint": "^4.3.9", "@smithy/middleware-retry": "^4.4.9", "@smithy/middleware-serde": "^4.2.5", "@smithy/middleware-stack": "^4.2.5", "@smithy/node-config-provider": "^4.3.5", "@smithy/node-http-handler": "^4.4.5", "@smithy/protocol-http": "^5.3.5", "@smithy/smithy-client": "^4.9.5", "@smithy/types": "^4.9.0", "@smithy/url-parser": "^4.2.5", "@smithy/util-base64": "^4.3.0", "@smithy/util-body-length-browser": "^4.2.0", "@smithy/util-body-length-node": "^4.2.1", "@smithy/util-defaults-mode-browser": "^4.3.8", "@smithy/util-defaults-mode-node": "^4.2.11", "@smithy/util-endpoints": "^3.2.5", "@smithy/util-middleware": "^4.2.5", "@smithy/util-retry": "^4.2.5", "@smithy/util-stream": "^4.5.6", "@smithy/util-utf8": "^4.2.0", "@smithy/util-waiter": "^4.2.5", "tslib": "^2.6.2" } }, "sha512-KxwZvdxdCeWK6o8mpnb+kk7Kgb8V+8AjTwSXUWH1UAD85B0tjdo1cSfE5zoR5fWGol4Ml5RLez12a6LPhsoTqA=="], @@ -5765,8 +5762,6 @@ "@aws-sdk/client-cognito-identity/@aws-sdk/types": ["@aws-sdk/types@3.973.9", "", { "dependencies": { "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-kuBfgQVdcz5Bmapc4A13YbpVw/pXkesfhetcFYwbntqas8sF41OHyd4o28+/TG2ZQdHBsv90Lsu5y6oitvYCdg=="], - "@aws-sdk/client-firehose/@smithy/util-utf8": ["@smithy/util-utf8@4.2.2", "", { "dependencies": { "@smithy/util-buffer-from": "^4.2.2", "tslib": "^2.6.2" } }, "sha512-75MeYpjdWRe8M5E3AW0O4Cx3UadweS+cwdXjwYGBW5h/gxxnbeZ877sLPX/ZJA9GVTlL/qG0dXP29JWFCD1Ayw=="], - "@aws-sdk/client-lambda/@aws-sdk/core": ["@aws-sdk/core@3.974.15", "", { "dependencies": { "@aws-sdk/types": "^3.973.9", "@aws-sdk/xml-builder": "^3.972.26", "@aws/lambda-invoke-store": "^0.2.2", "@smithy/core": "^3.24.5", "@smithy/signature-v4": "^5.4.5", "@smithy/types": "^4.14.2", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-UpA0rTGW/tHGITcCqHisbuuEPraYg9GG+mWmXjY5+RxZBMLGe6aL9oe0ix50LztwAcPIkGZLH0yWdMIkCM10hw=="], "@aws-sdk/client-lambda/@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.47", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.41", "@aws-sdk/credential-provider-http": "^3.972.43", "@aws-sdk/credential-provider-ini": "^3.972.46", "@aws-sdk/credential-provider-process": "^3.972.41", "@aws-sdk/credential-provider-sso": "^3.972.45", "@aws-sdk/credential-provider-web-identity": "^3.972.45", "@aws-sdk/types": "^3.973.9", "@smithy/core": "^3.24.5", "@smithy/credential-provider-imds": "^4.3.6", "@smithy/types": "^4.14.2", "tslib": "^2.6.2" } }, "sha512-HrId+C0DWA5qDIyLG64/kjUB2RNtPypxmABnIctK+TA1P1kHlOYoE/Wf5T5tKOMKgb08P7k/zNyhvfJ3lh5Oag=="], diff --git a/infra/lake.ts b/infra/lake.ts index a6125a1ee1..ccad966006 100644 --- a/infra/lake.ts +++ b/infra/lake.ts @@ -1,8 +1,8 @@ -import { domain } from "./stage" - -// Retired in favor of the R2 lake and pending removal. Pulumi reads forceDestroy -// and retainOnDelete from state at delete time, so these settings must deploy to -// every stage before the resources are removed from code. +// Retired in favor of the R2 lake. Nothing writes to it anymore, but the table, +// catalog and Athena workgroup stay while the anomalyco/anomaly inference and +// analytics dashboards still query its history. Pulumi reads forceDestroy and +// retainOnDelete from state at delete time, so keep those settings until the +// lake is removed. const current = aws.getCallerIdentityOutput({}) const partition = aws.getPartitionOutput({}) @@ -10,11 +10,6 @@ const region = aws.getRegionOutput({}) const tableBucketName = `opencode-${$app.stage}-lake` const glueCatalogName = "s3tablescatalog" -const glueCatalogArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:catalog` -const glueS3TablesCatalogArn = $interpolate`${glueCatalogArn}/${glueCatalogName}` -const glueS3TablesChildCatalogArn = $interpolate`${glueS3TablesCatalogArn}/${tableBucketName}` -const glueS3TablesDatabaseWildcardArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/${glueCatalogName}/${tableBucketName}/*` -const glueS3TablesTableWildcardArn = $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/${tableBucketName}/*/*` const s3TablesBucketWildcardArn = $interpolate`arn:${partition.partition}:s3tables:${region.region}:${current.accountId}:bucket/*` export const tableBucket = new aws.s3tables.TableBucket("LakeTableBucket", { @@ -22,7 +17,7 @@ export const tableBucket = new aws.s3tables.TableBucket("LakeTableBucket", { forceDestroy: true, }) -const s3TablesCatalog = new aws.cloudcontrol.Resource( +new aws.cloudcontrol.Resource( "LakeS3TablesCatalog", { typeName: "AWS::Glue::Catalog", @@ -64,7 +59,8 @@ const athenaResultsBucket = new aws.s3.Bucket( { retainOnDelete: false }, ) -const firehoseErrorBucket = new aws.s3.Bucket( +// Keep the archived delivery failures until their records have been reconciled. +new aws.s3.Bucket( "LakeFirehoseErrors", { bucket: `opencode-${$app.stage}-lake-firehose-errors`, @@ -89,200 +85,5 @@ new aws.athena.Workgroup("LakeAthenaWorkgroup", { }, }) -const firehoseRole = new aws.iam.Role("LakeFirehoseRole", { - assumeRolePolicy: aws.iam.getPolicyDocumentOutput({ - statements: [ - { - effect: "Allow", - actions: ["sts:AssumeRole"], - principals: [ - { - type: "Service", - identifiers: ["firehose.amazonaws.com"], - }, - ], - }, - ], - }).json, -}) - -const firehosePolicy = new aws.iam.RolePolicy("LakeFirehosePolicy", { - role: firehoseRole.id, - policy: aws.iam.getPolicyDocumentOutput({ - statements: [ - { - effect: "Allow", - actions: [ - "s3tables:ListTableBuckets", - "s3tables:GetTableBucket", - "s3tables:GetNamespace", - "s3tables:GetTable", - "s3tables:GetTableData", - "s3tables:GetTableMetadataLocation", - "s3tables:ListNamespaces", - "s3tables:ListTables", - "s3tables:PutTableData", - "s3tables:UpdateTableMetadataLocation", - ], - resources: ["*"], - }, - { - effect: "Allow", - actions: [ - "glue:GetCatalog", - "glue:GetCatalogs", - "glue:GetDatabase", - "glue:GetDatabases", - "glue:GetTable", - "glue:GetTables", - "glue:UpdateTable", - ], - resources: [ - glueCatalogArn, - glueS3TablesCatalogArn, - $interpolate`${glueS3TablesCatalogArn}/*`, - glueS3TablesDatabaseWildcardArn, - glueS3TablesTableWildcardArn, - $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:database/*`, - $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/*/*`, - $interpolate`arn:${partition.partition}:glue:${region.region}:${current.accountId}:table/${glueCatalogName}/*`, - ], - }, - { - effect: "Allow", - actions: [ - "s3:AbortMultipartUpload", - "s3:GetBucketLocation", - "s3:GetObject", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:PutObject", - ], - resources: [firehoseErrorBucket.arn, $interpolate`${firehoseErrorBucket.arn}/*`], - }, - { - effect: "Allow", - actions: ["lakeformation:GetDataAccess"], - resources: ["*"], - }, - ], - }).json, -}) - -const firehose = new aws.kinesis.FirehoseDeliveryStream( - "LakeFirehose", - { - name: `opencode-${$app.stage}-lake-ingest`, - destination: "iceberg", - icebergConfiguration: { - appendOnly: true, - bufferingInterval: 60, - bufferingSize: 1, - catalogArn: glueS3TablesChildCatalogArn, - processingConfiguration: { - enabled: true, - processors: [ - { - type: "MetadataExtraction", - parameters: [ - { parameterName: "JsonParsingEngine", parameterValue: "JQ-1.6" }, - { - parameterName: "MetadataExtractionQuery", - parameterValue: - '{destinationDatabaseName:._lake_database,destinationTableName:._lake_table,operation:(._lake_operation // "insert")}', - }, - ], - }, - ], - }, - roleArn: firehoseRole.arn, - s3BackupMode: "FailedDataOnly", - s3Configuration: { - roleArn: firehoseRole.arn, - bucketArn: firehoseErrorBucket.arn, - errorOutputPrefix: "errors/!{firehose:error-output-type}/", - }, - }, - }, - { dependsOn: [s3TablesCatalog, firehosePolicy] }, -) - export const lakeVpc = new sst.aws.Vpc("LakeVpc") export const lakeCluster = new sst.aws.Cluster("LakeCluster", { vpc: lakeVpc }) - -const ingestSecret = new random.RandomPassword("LakeIngestSecret", { length: 32 }) -export const ingestSecretSsm = new aws.ssm.Parameter("LakeIngestSecretSsm", { - name: $interpolate`/${$app.name}/${$app.stage}/lake/ingest/secret`, - type: "SecureString", - value: ingestSecret.result, -}) - -const ingestConfig = new sst.Linkable("LakeIngestConfig", { - properties: { - streamName: firehose.name, - secret: ingestSecret.result, - }, -}) - -const ingestService = new sst.aws.Service("LakeIngestService", { - cluster: lakeCluster, - architecture: "arm64", - cpu: "1 vCPU", - memory: "4 GB", - image: { - context: ".", - dockerfile: "packages/stats/server/Dockerfile", - }, - link: [ingestConfig], - permissions: [ - { - actions: ["firehose:PutRecord", "firehose:PutRecordBatch"], - resources: [firehose.arn], - }, - ], - scaling: { - min: $app.stage === "production" ? 2 : 1, - max: $app.stage === "production" ? 32 : 4, - cpuUtilization: 60, - memoryUtilization: 70, - }, - loadBalancer: { - domain: { - name: `lake.${domain}`, - dns: sst.cloudflare.dns(), - }, - rules: [ - { listen: "80/http", redirect: "443/https" }, - { listen: "443/https", forward: "3000/http" }, - ], - health: { - "3000/http": { - path: "/ready", - successCodes: "200-299", - }, - }, - }, - health: { - command: [ - "CMD-SHELL", - "bun --eval \"fetch('http://localhost:3000/health').then((r) => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))\"", - ], - interval: "30 seconds", - retries: 3, - startPeriod: "30 seconds", - timeout: "5 seconds", - }, - dev: { - command: "bun run start", - directory: "packages/stats/server", - url: "http://localhost:3000", - }, - wait: $app.stage === "production", -}) - -export const lakeIngest = new sst.Linkable("LakeIngest", { - properties: { - url: ingestService.url, - secret: ingestSecret.result, - }, -}) diff --git a/packages/stats/server/Dockerfile b/packages/stats/server/Dockerfile index 01a9004b7b..c42476e8d5 100644 --- a/packages/stats/server/Dockerfile +++ b/packages/stats/server/Dockerfile @@ -29,6 +29,4 @@ COPY --from=pruner /app/out/full/ ./ WORKDIR /app/packages/stats/server -EXPOSE 3000 - -CMD ["bun", "src/server.ts"] +CMD ["bun", "src/stat-sync.ts"] diff --git a/packages/stats/server/package.json b/packages/stats/server/package.json index 9bae345f81..5c30056b26 100644 --- a/packages/stats/server/package.json +++ b/packages/stats/server/package.json @@ -5,16 +5,15 @@ "private": true, "type": "module", "license": "MIT", - "main": "./src/server.ts", + "main": "./src/stat-sync.ts", "exports": { - ".": "./src/server.ts" + ".": "./src/stat-sync.ts" }, "scripts": { - "start": "bun src/server.ts", + "start": "bun src/stat-sync.ts", "typecheck": "tsgo --noEmit" }, "dependencies": { - "@aws-sdk/client-firehose": "3.933.0", "@effect/platform-node": "catalog:", "@opencode-ai/stats-core": "workspace:*", "effect": "catalog:", diff --git a/packages/stats/server/src/ingest.ts b/packages/stats/server/src/ingest.ts deleted file mode 100644 index eda662d989..0000000000 --- a/packages/stats/server/src/ingest.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { Buffer } from "node:buffer" -import { FirehoseClient, PutRecordBatchCommand } from "@aws-sdk/client-firehose" -import { Effect, Layer } from "effect" -import * as Context from "effect/Context" -import { Resource } from "sst/resource" - -const MAX_FIREHOSE_BATCH_SIZE = 500 -const MAX_FIREHOSE_ATTEMPTS = 3 -const LAKE_TYPE = /^([A-Za-z0-9_]+)\.([A-Za-z0-9_]+)$/ - -type IngestEvent = Record -type LakeRoute = { database: string; table: string } -type FirehoseRecord = { Data: Uint8Array } - -export class IngestError extends Error { - readonly _tag = "IngestError" - readonly failed: number - - constructor(input: { message: string; failed: number; cause?: unknown }) { - super(input.message, { cause: input.cause }) - this.name = "IngestError" - this.failed = input.failed - } -} - -export declare namespace Ingest { - export interface Service { - readonly write: (events: unknown[]) => Effect.Effect<{ records: number }, IngestError> - } -} - -export class Ingest extends Context.Service()("@opencode/stats/Ingest") { - static readonly layer: Layer.Layer = Layer.effect( - Ingest, - Effect.sync(() => { - const client = new FirehoseClient({}) - - const write = Effect.fn("Ingest.write")(function* (events: unknown[]) { - if (events.length === 0) return { records: 0 } - const counts = countRoutedEvents(events) - if (counts.unsupported > 0) { - yield* Effect.logWarning( - `lake ingest rejected ${JSON.stringify({ records: counts.records, unsupported: counts.unsupported })}`, - ) - return yield* Effect.fail( - new IngestError({ - message: "Unsupported lake event type", - failed: counts.unsupported, - }), - ) - } - if (counts.records === 0) return { records: 0 } - - let batch: FirehoseRecord[] = [] - let batches = 0 - let failed = 0 - - for (const event of events) { - if (!isRecord(event)) continue - const route = routeEvent(event) - if (!route) continue - batch.push(toFirehoseRecord(event, route)) - if (batch.length < MAX_FIREHOSE_BATCH_SIZE) continue - failed += yield* putRecords(client, Resource.LakeIngestConfig.streamName, batch) - batches++ - batch = [] - } - - if (batch.length > 0) { - failed += yield* putRecords(client, Resource.LakeIngestConfig.streamName, batch) - batches++ - } - - if (failed > 0) { - yield* Effect.logWarning(`lake ingest incomplete ${JSON.stringify({ records: counts.records, failed })}`) - return yield* Effect.fail(new IngestError({ message: "Failed to ingest all lake records", failed })) - } - - yield* Effect.logInfo(`lake ingest complete ${JSON.stringify({ records: counts.records, batches })}`) - return { records: counts.records } - }) - - return Ingest.of({ write }) - }), - ) -} - -const putRecords: ( - client: FirehoseClient, - streamName: string, - records: FirehoseRecord[], - attempt?: number, -) => Effect.Effect = Effect.fn("Ingest.putRecords")(function* ( - client, - streamName, - records, - attempt = 1, -) { - const result = yield* Effect.tryPromise({ - try: () => client.send(new PutRecordBatchCommand({ DeliveryStreamName: streamName, Records: records })), - catch: (cause) => - new IngestError({ message: "Failed to write lake records to Firehose", failed: records.length, cause }), - }).pipe( - Effect.tapError(() => - Effect.logWarning(`firehose batch write failed ${JSON.stringify({ records: records.length, attempt })}`), - ), - ) - const failed = - result.RequestResponses?.flatMap((item, index) => { - const record = records[index] - if (!item.ErrorCode || !record) return [] - return [record] - }) ?? [] - - if (failed.length === 0) return 0 - if (attempt >= MAX_FIREHOSE_ATTEMPTS) { - yield* Effect.logWarning( - `firehose batch failed ${JSON.stringify({ records: failed.length, attempts: MAX_FIREHOSE_ATTEMPTS })}`, - ) - return failed.length - } - - yield* Effect.logWarning( - `firehose batch retrying ${JSON.stringify({ records: failed.length, attempt: attempt + 1 })}`, - ) - yield* Effect.sleep(`${250 * 2 ** (attempt - 1)} millis`) - return yield* putRecords(client, streamName, failed, attempt + 1) -}) - -function countRoutedEvents(events: unknown[]) { - let records = 0 - let unsupported = 0 - for (const event of events) { - if (!isRecord(event)) continue - if (routeEvent(event)) records++ - else unsupported++ - } - return { records, unsupported } -} - -function isRecord(item: unknown): item is IngestEvent { - return Boolean(item) && typeof item === "object" && !Array.isArray(item) -} - -function routeEvent(event: IngestEvent): LakeRoute | undefined { - if (typeof event._datalake_key !== "string") return - const match = event._datalake_key.match(LAKE_TYPE) - if (!match?.[1] || !match[2]) return - return { - database: match[1], - table: match[2], - } -} - -function toFirehoseRecord(event: IngestEvent, route: LakeRoute): FirehoseRecord { - return { - Data: Buffer.from( - JSON.stringify({ - ...Object.fromEntries(Object.entries(event).filter(([key]) => key !== "_datalake_key")), - _lake_database: route.database, - _lake_table: route.table, - _lake_operation: "insert" as const, - }), - ), - } -} diff --git a/packages/stats/server/src/resource.d.ts b/packages/stats/server/src/resource.d.ts deleted file mode 100644 index 514b4d2afe..0000000000 --- a/packages/stats/server/src/resource.d.ts +++ /dev/null @@ -1,11 +0,0 @@ -import "sst/resource" - -declare module "sst/resource" { - export interface Resource { - LakeIngestConfig: { - secret: string - streamName: string - type: "sst.sst.Linkable" - } - } -} diff --git a/packages/stats/server/src/router.ts b/packages/stats/server/src/router.ts deleted file mode 100644 index e951118079..0000000000 --- a/packages/stats/server/src/router.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { Buffer } from "node:buffer" -import { timingSafeEqual } from "node:crypto" -import { Effect, Schema } from "effect" -import * as Semaphore from "effect/Semaphore" -import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http" -import { Resource } from "sst/resource" -import { Ingest } from "./ingest" -import { isShuttingDown } from "./shutdown" - -const MAX_CONCURRENT_INGEST_REQUESTS = 8 - -const IngestPayload = Schema.Struct({ - events: Schema.optional(Schema.Unknown), -}) - -export const Routes = HttpRouter.use((router) => - Effect.gen(function* () { - const ingestService = yield* Ingest - const ingestRequests = yield* Semaphore.make(MAX_CONCURRENT_INGEST_REQUESTS) - - yield* Effect.all( - [ - router.add("GET", "/health", () => json(200, { ok: true })), - router.add("GET", "/ready", () => json(isShuttingDown() ? 503 : 200, { ok: !isShuttingDown() })), - router.add("POST", "/", ingestRequests.withPermit(ingest(ingestService))), - ], - { discard: true }, - ) - }), -) - -const ingest = (ingestService: Ingest.Service) => - Effect.gen(function* () { - const request = yield* HttpServerRequest.HttpServerRequest - if (!isAuthorized(request.headers)) return yield* json(401, { ok: false, error: "Unauthorized" }) - - const payload = yield* HttpServerRequest.schemaBodyJson(IngestPayload).pipe( - Effect.match({ - onFailure: () => undefined, - onSuccess: (value) => value, - }), - ) - if (!payload) return yield* json(400, { ok: false, error: "Invalid JSON body" }) - - const events = Array.isArray(payload.events) ? payload.events : [] - if (events.length === 0) return yield* json(202, { ok: true, records: 0 }) - - return yield* ingestService.write(events).pipe( - Effect.flatMap((result) => json(202, { ok: true, records: result.records })), - Effect.catchTag("IngestError", (error) => - json(502, { ok: false, records: countRecords(events), failed: error.failed }), - ), - ) - }) - -function isAuthorized(headers: Record) { - const actual = Buffer.from(headers.authorization ?? headers.Authorization ?? "") - const expected = Buffer.from(`Bearer ${Resource.LakeIngestConfig.secret}`) - if (actual.length !== expected.length) return false - return timingSafeEqual(actual, expected) -} - -function countRecords(items: unknown[]) { - let records = 0 - for (const item of items) { - if (Boolean(item) && typeof item === "object" && !Array.isArray(item)) records++ - } - return records -} - -function json(status: number, body: Record) { - return HttpServerResponse.json(body, { status }).pipe(Effect.orDie) -} diff --git a/packages/stats/server/src/server.ts b/packages/stats/server/src/server.ts deleted file mode 100644 index 95104bc45e..0000000000 --- a/packages/stats/server/src/server.ts +++ /dev/null @@ -1,28 +0,0 @@ -import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer" -import * as NodeRuntime from "@effect/platform-node/NodeRuntime" -import { Config, Layer } from "effect" -import { HttpRouter } from "effect/unstable/http" -import { createServer } from "node:http" -import { Ingest } from "./ingest" -import { Routes } from "./router" -import { registerShutdownSignalHandlers } from "./shutdown" - -registerShutdownSignalHandlers() - -const ServerLive = NodeHttpServer.layerConfig( - () => createServer(), - Config.all({ - port: Config.number("PORT").pipe(Config.withDefault(3000)), - host: Config.string("HOST").pipe(Config.withDefault("0.0.0.0")), - }), -) - -const runtimeLayer = Ingest.layer -const programLayer = Routes.pipe(Layer.provide(runtimeLayer)) -const main = Layer.launch( - HttpRouter.serve(programLayer, { - disableLogger: true, - }).pipe(Layer.provideMerge(ServerLive)), -) - -NodeRuntime.runMain(main, { disableErrorReporting: true }) diff --git a/packages/stats/server/src/shutdown.ts b/packages/stats/server/src/shutdown.ts deleted file mode 100644 index 43938b3268..0000000000 --- a/packages/stats/server/src/shutdown.ts +++ /dev/null @@ -1,17 +0,0 @@ -let shuttingDown = false -let signalHandlersRegistered = false - -export function isShuttingDown() { - return shuttingDown -} - -export function registerShutdownSignalHandlers() { - if (signalHandlersRegistered) return - signalHandlersRegistered = true - process.once("SIGTERM", markShuttingDown) - process.once("SIGINT", markShuttingDown) -} - -function markShuttingDown() { - shuttingDown = true -} diff --git a/sst.config.ts b/sst.config.ts index b245a51324..41321005bf 100644 --- a/sst.config.ts +++ b/sst.config.ts @@ -30,7 +30,6 @@ export default $config({ async run() { const stage = await import("./infra/stage.js") await import("./infra/app.js") - const lake = stage.deployAws ? await import("./infra/lake.js") : undefined const stats = stage.deployAws ? await import("./infra/stats.js") : undefined const { stat } = await import("./infra/console.js") await import("./infra/enterprise.js") @@ -41,12 +40,6 @@ export default $config({ return { StatWorkerUrl: stat.url, ...(stats ? { StatsUrl: stats.app.url } : {}), - ...(lake - ? { - LakeUrl: lake.lakeIngest.properties.url, - LakeSecretSsm: lake.ingestSecretSsm.name, - } - : {}), AwsStage: stage.awsStage, } }, From cff9078bb31b2ed6e114446cb17b883586ddd0ed Mon Sep 17 00:00:00 2001 From: "opencode-agent[bot]" Date: Thu, 1 Oct 2026 21:16:11 +0000 Subject: [PATCH 009/105] chore: update nix node_modules hashes --- nix/hashes.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nix/hashes.json b/nix/hashes.json index b4cef486b6..f739950638 100644 --- a/nix/hashes.json +++ b/nix/hashes.json @@ -1,8 +1,8 @@ { "nodeModules": { - "x86_64-linux": "sha256-fAqsf8Zgjz9oinwnFmlDbnT8hW61MfP1aJgXDSWFHLg=", - "aarch64-linux": "sha256-Ite4U+M3tagfb/0f33XwNdXZRpbQDFD5g1SuKQMU9wo=", - "aarch64-darwin": "sha256-53v/BWVbpOFQqP+Wc906FWOih0xKbQgiumGbqmIS1fo=", - "x86_64-darwin": "sha256-tjWqIKUiwHg4goIb/aHy3EOEvYcjjK4ft0GHPiPhVbY=" + "x86_64-linux": "sha256-3xhjOXe9XsiC9jMKlIf8y0uRKI2a/PIKTdmsTeV/uFE=", + "aarch64-linux": "sha256-yU83HnXa2o0flViYykRj96skrH/MPX1FXQcmNM2itFM=", + "aarch64-darwin": "sha256-7B4h5fGOvFuFu5aa5uzzzkLb/98kG/oxr5uYJebFJrI=", + "x86_64-darwin": "sha256-yro96TsJWCnvOyhaGF6HEor6MtV+RkBWSiMw8n0UiOw=" } } From a79ecfe109294909a239c98fb89f02979d5aa10b Mon Sep 17 00:00:00 2001 From: Adam <2363879+adamdotdevin@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:41:08 -0500 Subject: [PATCH 010/105] feat(stats): add canonical redirects and agent-readable data formats --- .../app/src/lib/content-negotiation.ts | 13 + packages/console/app/src/lib/stats-proxy.ts | 24 +- .../opencode/test/cli/run/run-process.test.ts | 3 +- .../stats/app/src/component/agent-meta.tsx | 40 + .../app/src/component/chart-data-table.tsx | 28 + .../src/component/model-compare-detail.tsx | 50 +- packages/stats/app/src/context/i18n.tsx | 15 +- packages/stats/app/src/i18n.ts | 35 + packages/stats/app/src/i18n/ar.ts | 25 + packages/stats/app/src/i18n/br.ts | 26 + packages/stats/app/src/i18n/da.ts | 26 + packages/stats/app/src/i18n/de.ts | 27 + packages/stats/app/src/i18n/es.ts | 28 + packages/stats/app/src/i18n/fr.ts | 27 + packages/stats/app/src/i18n/it.ts | 27 + packages/stats/app/src/i18n/ja.ts | 26 + packages/stats/app/src/i18n/ko.ts | 25 + packages/stats/app/src/i18n/no.ts | 26 + packages/stats/app/src/i18n/pl.ts | 28 + packages/stats/app/src/i18n/ru.ts | 26 + packages/stats/app/src/i18n/th.ts | 25 + packages/stats/app/src/i18n/tr.ts | 27 + packages/stats/app/src/i18n/uk.ts | 26 + packages/stats/app/src/i18n/zh.ts | 23 + packages/stats/app/src/i18n/zht.ts | 23 + packages/stats/app/src/lib/agent-formats.ts | 658 +++++++++++++ packages/stats/app/src/lib/format.ts | 49 + packages/stats/app/src/lib/language.ts | 19 + packages/stats/app/src/lib/methodology.ts | 10 + packages/stats/app/src/lib/page-data.ts | 151 +++ packages/stats/app/src/lib/summaries.ts | 51 + packages/stats/app/src/middleware.ts | 162 +++ .../stats/app/src/routes/[lab]/[model].tsx | 118 ++- packages/stats/app/src/routes/[lab]/index.tsx | 247 +++-- .../app/src/routes/breadcrumb-select.tsx | 1 - .../stats/app/src/routes/compare-cards.tsx | 4 +- .../stats/app/src/routes/compare/index.tsx | 1 + packages/stats/app/src/routes/index.css | 203 +++- packages/stats/app/src/routes/index.tsx | 924 ++++++++++-------- .../stats/app/src/routes/model-catalog.ts | 53 +- .../stats/app/src/routes/section-heading.tsx | 17 +- packages/stats/app/src/routes/stats-cache.ts | 3 +- packages/stats/app/src/routes/stats-shell.tsx | 1 + packages/stats/app/test/agent-formats.test.ts | 48 + packages/stats/app/vite.config.ts | 2 +- 45 files changed, 2707 insertions(+), 664 deletions(-) create mode 100644 packages/console/app/src/lib/content-negotiation.ts create mode 100644 packages/stats/app/src/component/agent-meta.tsx create mode 100644 packages/stats/app/src/component/chart-data-table.tsx create mode 100644 packages/stats/app/src/lib/agent-formats.ts create mode 100644 packages/stats/app/src/lib/format.ts create mode 100644 packages/stats/app/src/lib/methodology.ts create mode 100644 packages/stats/app/src/lib/page-data.ts create mode 100644 packages/stats/app/src/lib/summaries.ts create mode 100644 packages/stats/app/src/middleware.ts create mode 100644 packages/stats/app/test/agent-formats.test.ts diff --git a/packages/console/app/src/lib/content-negotiation.ts b/packages/console/app/src/lib/content-negotiation.ts new file mode 100644 index 0000000000..ee27a03d83 --- /dev/null +++ b/packages/console/app/src/lib/content-negotiation.ts @@ -0,0 +1,13 @@ +// Shared by the console proxy and the stats app so both agree on which requests get Markdown. +export function prefersMarkdown(accept: string | null | undefined) { + if (!accept) return false + const weights = new Map( + accept.split(",").map((part) => { + const fields = part.split(";").map((value) => value.trim().toLowerCase()) + const quality = fields.slice(1).find((field) => field.startsWith("q=")) + return [fields[0] ?? "", quality ? Number(quality.slice(2)) : 1] as const + }), + ) + const markdown = Math.max(weights.get("text/markdown") ?? 0, weights.get("text/x-markdown") ?? 0) + return markdown > 0 && markdown >= (weights.get("text/html") ?? 0) +} diff --git a/packages/console/app/src/lib/stats-proxy.ts b/packages/console/app/src/lib/stats-proxy.ts index b8db97c063..7a2bb98cc5 100644 --- a/packages/console/app/src/lib/stats-proxy.ts +++ b/packages/console/app/src/lib/stats-proxy.ts @@ -1,5 +1,6 @@ import type { APIEvent } from "@solidjs/start/server" import { Resource, waitUntil } from "@opencode-ai/console-resource" +import { prefersMarkdown } from "~/lib/content-negotiation" import { LOCALE_HEADER, cookie, localeFromRequest, route, tag } from "~/lib/language" const dataPath = "/data" @@ -40,8 +41,12 @@ export async function statsProxy(evt: APIEvent) { method: req.method, headers: requestHeaders, body: req.body, + // Pass canonical redirects from the stats app to the client instead of following them here. + redirect: "manual", }) + const location = response.headers.get("location") + if (location) return withPublicLocation(response, new URL(location, targetUrl), targetUrl) if (!response.headers.get("content-type")?.includes("text/html")) return response const headers = new Headers(response.headers) @@ -49,7 +54,7 @@ export async function statsProxy(evt: APIEvent) { headers.delete("content-length") headers.delete("etag") headers.delete("set-cookie") - appendVary(headers, "Accept-Language", "Cookie", LOCALE_HEADER) + appendVary(headers, "Accept", "Accept-Language", "Cookie", LOCALE_HEADER) const result = new Response(rewriteStatsHtml(await response.text()), { status: response.status, @@ -73,6 +78,14 @@ export function statsRedirect(evt: APIEvent) { }) } +// A redirect to the stats origin itself would send clients past the proxy, so keep only the path. +function withPublicLocation(response: Response, location: URL, upstream: URL) { + if (location.origin !== upstream.origin) return response + const headers = new Headers(response.headers) + headers.set("location", `${location.pathname}${location.search}${location.hash}`) + return new Response(response.body, { status: response.status, statusText: response.statusText, headers }) +} + function rewriteStatsHtml(html: string) { return html.replaceAll('"/_build/', `"${dataPath}/_build/`).replaceAll("'/_build/", `'${dataPath}/_build/`) } @@ -120,7 +133,7 @@ function redirectToLocalizedData(request: Request, url: URL, locale: ReturnType< Location: next.toString(), }) headers.append("set-cookie", cookie(locale)) - appendVary(headers, "Accept-Language", "Cookie", LOCALE_HEADER) + appendVary(headers, "Accept", "Accept-Language", "Cookie", LOCALE_HEADER) return new Response(null, { status: 308, @@ -130,6 +143,8 @@ function redirectToLocalizedData(request: Request, url: URL, locale: ReturnType< function acceptsHtml(request: Request) { const accept = request.headers.get("accept") + // The stats app answers Markdown requests itself, so keep them out of the HTML cache and locale redirect. + if (prefersMarkdown(accept)) return false return !accept || accept.includes("text/html") || accept.includes("*/*") } @@ -140,7 +155,10 @@ function isDataBypassPath(pathname: string) { pathname.startsWith(`${dataPath}/_server`) || pathname === `${dataPath}/banner.jpg` || pathname === `${dataPath}/banner.png` || - pathname === `${dataPath}/sitemap.xml` + pathname === `${dataPath}/sitemap.xml` || + pathname.endsWith("/llms.txt") || + pathname.endsWith(".md") || + pathname.endsWith(".json") ) } diff --git a/packages/opencode/test/cli/run/run-process.test.ts b/packages/opencode/test/cli/run/run-process.test.ts index d2d4bae879..c6ccdc80cd 100644 --- a/packages/opencode/test/cli/run/run-process.test.ts +++ b/packages/opencode/test/cli/run/run-process.test.ts @@ -67,7 +67,8 @@ describe("opencode run (non-interactive subprocess)", () => { // makes the SDK call surface an error promptly so the process exits nonzero. // We assert nonzero exit AND wall-clock under the harness timeout — a hang // would expire the timeout and produce a different (signal-killed) failure. - cliIt.concurrent( + // Keep competing CLI startups out of this wall-clock assertion on busy CI runners. + cliIt.live( "exits nonzero promptly when the model is unknown (regression for #27371)", ({ opencode }) => Effect.gen(function* () { diff --git a/packages/stats/app/src/component/agent-meta.tsx b/packages/stats/app/src/component/agent-meta.tsx new file mode 100644 index 0000000000..39bf6bf94c --- /dev/null +++ b/packages/stats/app/src/component/agent-meta.tsx @@ -0,0 +1,40 @@ +import { Link } from "@solidjs/meta" +import { Show } from "solid-js" +import { getRequestEvent } from "solid-js/web" +import { jsonUrl, markdownUrl } from "../lib/language" + +export function JsonLd(props: { data: object }) { + // Escape "<" so values from the catalog can never close the script element. + return + +

model.name).join(", ")}`}> - Compare - - AI models + Compare AI models + : {props.models.map((model) => model.name).join(" vs ")}

+ [point.date, props.formatValue(props.value(point))])} + /> ) @@ -1033,17 +1046,23 @@ function MetricCard(props: { label: string; value: string; detail?: string; stat } function PeerRow(props: { peer: ModelPeerEntry; active: boolean; catalogLabs: readonly string[] }) { + const i18n = useI18n() const language = useLanguage() const hasProvider = () => isKnownCatalogLab(props.peer.provider, props.catalogLabs) return (
  • - {String(props.peer.rank).padStart(2, "0")} + {String(props.peer.rank).padStart(2, "0")}} + > + {String(props.peer.rank).padStart(2, "0")} + @@ -1053,10 +1072,13 @@ function PeerRow(props: { peer: ModelPeerEntry; active: boolean; catalogLabs: re {props.peer.model} - {props.peer.author} + {formatCatalogLabName(props.peer.provider)} - {formatTokens(props.peer.tokens)} + + {formatTokens(props.peer.tokens)} + {i18n.t("format.tokens")} +
  • ) diff --git a/packages/stats/app/src/routes/[lab]/index.tsx b/packages/stats/app/src/routes/[lab]/index.tsx index 980602d826..1a52f40be8 100644 --- a/packages/stats/app/src/routes/[lab]/index.tsx +++ b/packages/stats/app/src/routes/[lab]/index.tsx @@ -1,8 +1,6 @@ import { Meta, Title } from "@solidjs/meta" import { ProviderIcon } from "@opencode-ai/ui/provider-icon" import { - getStatsLabData, - getStatsHomeData, type LabUsageModelEntry, type MarketDay, type ModelUsagePoint, @@ -11,22 +9,24 @@ import { import { createAsync, query, useParams } from "@solidjs/router" import { createMemo, createSignal, createUniqueId, For, onMount, Show, type JSX } from "solid-js" import { getRequestEvent } from "solid-js/web" +import { breadcrumbList, FormatLinks, JsonLd, LastModified } from "../../component/agent-meta" +import { ChartDataTable } from "../../component/chart-data-table" import { LocaleLinks } from "../../component/locale-links" import { NotFoundMeta } from "../../component/not-found-meta" import { useI18n } from "../../context/i18n" import { useLanguage } from "../../context/language" +import { fromFirstUsage } from "../../lib/format" import { localizedUrl } from "../../lib/language" +import { loadLabPage, type LabPageData } from "../../lib/page-data" +import { labSummary } from "../../lib/summaries" import { catalogModelPath, catalogSlug, - findModelCatalogLab, formatCatalogLabName, - loadModelCatalog, type ModelCatalogEntry, type ModelCatalogLab, } from "../model-catalog" import { SectionHeading } from "../section-heading" -import { runStatsEffect } from "../../stats-runtime" import { setStatsPageCacheHeaders } from "../stats-cache" import { ComparisonCardsSection, modelRefFromCatalog, uniqueComparisonPairs } from "../compare-cards" import { BreadcrumbSelect } from "../breadcrumb-select" @@ -44,32 +44,11 @@ import { const statsUnfurlPath = "banner.png" -type RelatedCatalogLab = Pick & { - models: Pick[] -} - -type LabPageData = { - lab: ModelCatalogLab | null - labs: RelatedCatalogLab[] - market: MarketDay[] - stats: StatsLabData | null -} +type RelatedCatalogLab = LabPageData["labs"][number] const getLabPageData = query(async (labParam: string) => { "use server" - const [catalog, home] = await Promise.all([loadModelCatalog(), runStatsEffect(getStatsHomeData())]) - const lab = findModelCatalogLab(catalog, labParam) ?? null - return { - lab, - labs: catalog.labs.map((entry) => ({ - id: entry.id, - name: entry.name, - description: entry.description, - models: entry.models.map((model) => ({ name: model.name })), - })), - market: home.market["2M"], - stats: lab ? await runStatsEffect(getStatsLabData(lab.id)) : null, - } satisfies LabPageData + return loadLabPage(labParam) }, "getStatsLabPageData") type LabModelTooltipState = { @@ -90,11 +69,16 @@ export default function StatsLab() { const page = createAsync(() => getLabPageData(labParam())) const lab = createMemo(() => page()?.lab) const stats = createMemo(() => page()?.stats) + const modelPaths = createMemo(() => page()?.modelPaths ?? {}) const githubStars = createAsync(() => getGitHubStars()) const [themePreference, setThemePreference] = createSignal("system") const labName = createMemo(() => lab()?.name ?? formatCatalogLabName(labParam())) const labTitle = createMemo(() => i18n.t("lab.title", { lab: labName() })) const labDescription = createMemo(() => i18n.t("lab.description", { lab: labName() })) + const labMetaDescription = createMemo(() => { + const data = page() + return (data && labSummary(language.locale(), data)) ?? labDescription() + }) const labPath = createMemo(() => `/data/${lab()?.id ?? labParam()}`) const labUrl = createMemo(() => localizedUrl(language.locale(), labPath())) const statsUnfurlUrl = new URL(statsUnfurlPath, localizedUrl("en", "/data/")).toString() @@ -108,6 +92,7 @@ export default function StatsLab() { { href: `${import.meta.env.BASE_URL}#top-models`, label: i18n.t("nav.topModels") }, { href: `${import.meta.env.BASE_URL}#market-share`, label: i18n.t("nav.marketShare") }, { href: `${import.meta.env.BASE_URL}#geo-breakdown`, label: i18n.t("nav.geoBreakdown") }, + { href: `${import.meta.env.BASE_URL}#methodology`, label: i18n.t("methodology.title") }, ]) const updateThemePreference = (preference: ThemePreference) => { applyThemePreference(preference) @@ -129,24 +114,57 @@ export default function StatsLab() { {/* Server-rendered head tags are never removed, so render them once data has loaded. */} {labTitle()} - + }> - - - - - - - - - - - - - - - - + {(data) => ( + <> + + + + + + + + + + + + + + + + + + + modelPaths()[model.id] === catalogModelPath(model)) + .map((model, index) => ({ + "@type": "ListItem", + position: index + 1, + name: model.name, + url: localizedUrl(language.locale(), catalogModelPath(model)), + })), + }, + }} + /> + + )}
    - +
    + [ + point.date, + formatTokens(point.tokens), + formatUsers(point.users), + ])} + /> ) } -function LabModelsSection(props: { lab: ModelCatalogLab; usage: LabUsageModelEntry[] }) { +function LabModelsSection(props: { + lab: ModelCatalogLab + usage: LabUsageModelEntry[] + paths: LabPageData["modelPaths"] +}) { const i18n = useI18n() const [activeTooltip, setActiveTooltip] = createSignal() const usageBySlug = createMemo(() => new Map(props.usage.map((item) => [item.slug, item]))) @@ -529,38 +560,47 @@ function LabModelsSection(props: { lab: ModelCatalogLab; usage: LabUsageModelEnt