diff --git a/.dockerignore b/.dockerignore index cfb63f532..630b8eb4f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,3 +15,5 @@ __pycache__ huggingface *.log .system_generated + +**/node_modules diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..127e70cb4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +dev/dashboard/package-lock.json linguist-generated=true diff --git a/.gitignore b/.gitignore index 4f732ed3f..a5b84aa9a 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,7 @@ site/.hugo_build.lock # The site serves this on purpose (design 016). !site/static/llms.txt + +# Dashboard build +node_modules/ +src/server/dashboard/static/app.js diff --git a/Makefile b/Makefile index be38e5581..99ce5ed2e 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ HOST ?= 127.0.0.1 PORT ?= 9003 # The fully qualified base URL used by local CLI tools and clients BASE_URL ?= http://$(HOST):$(PORT) -UNIT_TESTS ?= tests.test_session_lifecycle tests.test_proto_codec tests.test_forward_only tests.test_fft_batch_failure tests.test_api_server_paths tests.test_accel_timeslicer tests.test_trainer_optimizer_correctness tests.test_trainer_logprob_chunks tests.test_worker_manager tests.test_scheduler_worker_manager tests.test_estimator tests.test_redis_store tests.test_cluster_eval_script tests.test_delta_weight_sync tests.test_diffing_backends tests.test_sampler_weight_rotation tests.test_commands tests.test_weight_sync_config tests.test_exclusive_trainer tests.test_delete_model tests.test_automodel_worker tests.test_worker_thread_device tests.test_trainer_data_parallel tests.test_sampler_batch_limits tests.test_openrl_status tests.test_skill_docs tests.test_trainer_ready tests.test_sampling_store tests.test_lora_snapshots tests.test_pool_accept tests.test_sampling_dispatcher +UNIT_TESTS ?= tests.test_session_lifecycle tests.test_proto_codec tests.test_forward_only tests.test_fft_batch_failure tests.test_api_server_paths tests.test_accel_timeslicer tests.test_trainer_optimizer_correctness tests.test_trainer_logprob_chunks tests.test_worker_manager tests.test_scheduler_worker_manager tests.test_estimator tests.test_redis_store tests.test_cluster_eval_script tests.test_delta_weight_sync tests.test_diffing_backends tests.test_sampler_weight_rotation tests.test_commands tests.test_weight_sync_config tests.test_exclusive_trainer tests.test_delete_model tests.test_automodel_worker tests.test_worker_thread_device tests.test_trainer_data_parallel tests.test_sampler_batch_limits tests.test_openrl_status tests.test_skill_docs tests.test_trainer_ready tests.test_sampling_store tests.test_lora_snapshots tests.test_pool_accept tests.test_sampling_dispatcher tests.test_worker_record tests.test_telemetry_faults tests.test_dashboard tests.test_dashboard_facts # Only forward BASE_URL to e2e when the user supplied it. The Makefile default # is for local CLI usage; e2e should start its own backend by default. TRAINING_TEST_BASE_URL ?= $(if $(filter environment command line,$(origin BASE_URL)),$(BASE_URL),) @@ -366,3 +366,11 @@ push-vm: # Pull changes from the remote VM back to the local workspace pull-vm: rsync -avz --exclude '.git' --exclude '.venv' --exclude '__pycache__' --exclude '*.pyc' --exclude '.DS_Store' --exclude 'scratch' $(REMOTE_HOST):~/open-rl/ ./ + +# Native dashboard assets are built once; Python serves the resulting static files. +.PHONY: dashboard-build dashboard +dashboard-build: + npm --prefix dev/dashboard run build + +dashboard: dashboard-build + uv run --extra cluster python -m server.dashboard --port 8080 diff --git a/dev/dashboard/build.mjs b/dev/dashboard/build.mjs new file mode 100644 index 000000000..40a29818e --- /dev/null +++ b/dev/dashboard/build.mjs @@ -0,0 +1,20 @@ +import { build } from "esbuild"; +import { compile, compileModule } from "svelte/compiler"; +import { readFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = fileURLToPath(new URL("../../", import.meta.url)); +await build({ + entryPoints: [root + "src/server/dashboard/ui/index.js"], + outfile: root + "src/server/dashboard/static/app.js", + bundle: true, minify: true, format: "esm", conditions: ["browser"], + nodePaths: [fileURLToPath(new URL("node_modules", import.meta.url))], + plugins: [{ name: "svelte", setup(build) { + build.onLoad({ filter: /\.svelte(?:\.js)?$/ }, async ({ path }) => { + const source = await readFile(path, "utf8"), options = { filename: path, generate: "client", dev: false }; + const { js, warnings } = path.endsWith(".svelte.js") ? compileModule(source, options) : compile(source, options); + return { contents: js.code, resolveDir: dirname(path), warnings: warnings.map((w) => ({ text: `${w.code}: ${w.message}` })) }; + }); + } }], +}); diff --git a/dev/dashboard/package-lock.json b/dev/dashboard/package-lock.json new file mode 100644 index 000000000..e16736eca --- /dev/null +++ b/dev/dashboard/package-lock.json @@ -0,0 +1,701 @@ +{ + "name": "openrl-dashboard", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "openrl-dashboard", + "devDependencies": { + "esbuild": "0.28.1", + "svelte": "5.57.2" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@sveltejs/acorn-typescript": { + "version": "1.0.13", + "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.13.tgz", + "integrity": "sha512-wgKggnhZVL9Bfx1OaKKTrYY9BFRk6C8UAkQNUcIv1+llzYrIqy+RZm5HPKzn0NpEBvTVhTqB4kQyllZywsRBRQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^8.9.0" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/acorn": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.19.0.tgz", + "integrity": "sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/aria-query": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.1.tgz", + "integrity": "sha512-Z/ZeOgVl7bcSYZ/u/rh0fOpvEpq//LZmdbkXyc7syVzjPAhfOa9ebsdTSjEBDU4vs5nC98Kfduj1uFo0qyET3g==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/axobject-query": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", + "integrity": "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devalue": { + "version": "5.9.4", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.9.4.tgz", + "integrity": "sha512-sPAT4pztbu6586/hrhOnMKS17IJrvg12mXiSPSS3W5qDeN2RGgvZ0diZCm31dBbnevfVmujNO3IM2wrS4Y2Rhg==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/esm-env": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.2.tgz", + "integrity": "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/esrap": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.4.0.tgz", + "integrity": "sha512-CQPcneEHw/czj32uwQC52nYnax8H4SXt25FCM+LIwGZ49cMIxpOj3bP8wmzcI+Z0HIy5KkClaPhRLfCySzwHPA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.15" + }, + "peerDependencies": { + "@typescript-eslint/types": "^8.2.0" + }, + "peerDependenciesMeta": { + "@typescript-eslint/types": { + "optional": true + } + } + }, + "node_modules/is-reference": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.3.tgz", + "integrity": "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.6" + } + }, + "node_modules/locate-character": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz", + "integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==", + "dev": true, + "license": "MIT" + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/svelte": { + "version": "5.57.2", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.57.2.tgz", + "integrity": "sha512-1YiPCAXT0Ct6g1vOMmSLa0Nsc3QRMvtDF32ihgglGVrugIuIK38RyDiFNXVDhtQ81TJjSO57NdMw6DyYLqSGDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.5", + "@jridgewell/sourcemap-codec": "^1.6.0", + "@sveltejs/acorn-typescript": "^1.0.13", + "@types/estree": "^1.0.9", + "acorn": "^8.18.0", + "aria-query": "5.3.1", + "axobject-query": "^4.1.0", + "clsx": "^2.1.1", + "devalue": "^5.9.2", + "esm-env": "^1.2.1", + "esrap": "^2.3.6", + "is-reference": "^3.0.3", + "locate-character": "^3.0.0", + "magic-string": "^0.30.11", + "zimmerframe": "^1.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/zimmerframe": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/zimmerframe/-/zimmerframe-1.1.5.tgz", + "integrity": "sha512-msJxIvYDYcoNL+PJsu+7qmpDWsYmAxTY+2TNYXXF0hzBzBk0BMecOqDOG/EckUoKCuKwObfbugIl8QpqHDXeFA==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/dev/dashboard/package.json b/dev/dashboard/package.json new file mode 100644 index 000000000..8a8e888af --- /dev/null +++ b/dev/dashboard/package.json @@ -0,0 +1,12 @@ +{ + "name": "openrl-dashboard", + "private": true, + "type": "module", + "scripts": { + "build": "node build.mjs" + }, + "devDependencies": { + "esbuild": "0.28.1", + "svelte": "5.57.2" + } +} diff --git a/docs/configuration.md b/docs/configuration.md index 5d16bedbe..a549942f7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -100,3 +100,34 @@ On Kubernetes the release bundles set these variables. The API server runs with each trainer and sampler, and the OpenRL scheduler starts their pods. The API server writes each worker's environment into the `Workload`'s pod template, so there are no worker deployments to configure by hand. + +## Run telemetry + +Trainers, samplers and the API server record each run in Redis. +`python -m server.dashboard` serves the records read only under +`/api/v1/inspect`; `/docs` describes each route. Design Doc 018 explains them. + +| Env var | Default | What it does | +| --- | --- | --- | +| `OPEN_RL_OPS_SAMPLE_LIMIT` | `2000` | Newest command records kept per run, for trainer ops and samples separately. The scheduler passes it to worker pods. | +| `OPEN_RL_OPS_TTL_DAYS` | `7` | Days a record lives after its last write. `0` keeps records. The scheduler passes it to worker pods. | +| `OPEN_RL_INSPECTION_API` | unset | `true` mounts `/api/v1/inspect` on the API server too. It lists every run, so leave it off on a server clients reach. On GKE it reads nodes cluster-wide, which `k8s/deploy/base` does not grant; apply `k8s/deploy/dashboard`, whose RBAC binds the same service account. | +| `OPEN_RL_PROMETHEUS_URL` | unset | Prometheus for GPU and vLLM metrics. Unset, the dashboard uses Managed Prometheus on GKE. | +| `OPEN_RL_GKE_PROJECT` / `OPEN_RL_GKE_CLUSTER` / `OPEN_RL_GKE_LOCATION` | from the metadata server | The cluster whose metrics and logs the dashboard reads. `OPEN_RL_GKE_TELEMETRY=false` turns GKE reads off. | +| `OPEN_RL_LOG_VIEWS` | the project | Comma-separated Cloud Logging views to read run logs from, such as a long-retention bucket's view. | +| `OPEN_RL_CLIENT_RUNS_ROOT` | `/mnt/shared` | Where the dashboard looks for each run's client directory, three levels deep. | + +### Client metrics and logs + +The dashboard reads a run's client metrics and logs from the directory the +tinker-cookbook client writes, its `log_path`. The client sends nothing extra. +On GKE, set each run's `log_path` on the shared volume, such as +`/mnt/shared//`, which the API server also mounts at +`/mnt/shared`. A run whose directory the server cannot read has no client +metrics or logs. + +A directory is a run's when its `checkpoints.jsonl` names `tinker:///`. +Before the first checkpoint, it is a run's when its `config.json` was written +within 60 seconds of the run's creation and names the run's base model. When +several directories match a run and the time does not single one out, none +is used. diff --git a/docs/dashboard.md b/docs/dashboard.md new file mode 100644 index 000000000..0f953917f --- /dev/null +++ b/docs/dashboard.md @@ -0,0 +1,76 @@ +# Cluster dashboard and inspection API + +`/dashboard` is an operator UI over a read-only JSON API under +`/api/v1/inspect`. Agents use the same routes; `/docs` and `/openapi.json` +describe them, and [Design Doc 018](designs/018-run-diagnostics.md) explains +what each answers. + +## Run it + +```sh +npm --prefix dev/dashboard ci # once; installs Svelte and esbuild +make dashboard # builds static/app.js and serves on port 8080 +``` + +`make dashboard` runs `python -m server.dashboard`. Point it at the API +server's Redis with `REDIS_URL`; without it the dashboard has its own empty +store. It reads Kubernetes in `OPEN_RL_WORKER_NAMESPACE`, else +`POD_NAMESPACE`, else `openrl-system`. Run `make dashboard-build` after changing +`src/server/dashboard/ui` and before `uv build`. + +On GKE, `kubectl apply -k k8s/deploy/dashboard` deploys it into +`openrl-system` with the API server's image, which builds the UI in a Node +stage. To open it from a laptop: + +```sh +kubectl -n openrl-system port-forward svc/open-rl-dashboard 8080:8000 +``` + +Then open `http://127.0.0.1:8080/dashboard`. + +The API lists every run in the fleet, so the API server does not mount it by +default. For a trusted local server, run `make dashboard-build`, then +`OPEN_RL_INSPECTION_API=true make server`, and open +`http://127.0.0.1:9003/dashboard`. + +Logs need Cloud Logging, nodes and pods need Kubernetes, and GPU and sampler +charts need a metrics backend. A source that cannot be read shows as an +error, never as an empty cluster. [Run telemetry](configuration.md#run-telemetry) +lists the settings that choose the sources. Logs come from the project's +default buckets, usually kept 30 days. To keep them longer, route the +namespace's logs to a bucket and set `OPEN_RL_LOG_VIEWS` to its view, which +needs `roles/logging.viewAccessor`. + +## Pages + +- **Overview** lists runs with a status label, search, and status filters. +- **Run** shows status, settings, an activity timeline with GPU rows and duty, + metrics by step, sampler charts, and logs. Click the timeline to open the + logs around that time. +- **Nodes** shows which process held which GPU and when, each GPU's duty cycle, + and GPU utilization and memory. Drag or press A/D to pan; use Ctrl/⌘ + wheel, + pinch, or W/S to zoom. +- **Health** lists pod and node problems and sources that could not be read. + +## Lifecycle + +Each run in `/inventory` has a `lifecycle`, which says where it is, not whether +it is healthy: + +- **ended**: the API server marked it finished, its workers stopped, or it has + no workers and no live session. +- **waiting**: a worker is not placed yet, or the server has not created one + for a live session. +- **placed**: every worker is placed. +- **active**: the scheduler cannot be read, so placement is unknown. + +## Client metrics and logs + +The Run page charts the metrics a tinker-cookbook client writes to +`metrics.jsonl` in its `log_path`, and its logs' `client` source pages the +client's `logs.log`. The client sends nothing extra. On GKE, set the run's +`log_path` on the shared volume, such as `/mnt/shared//`, which +`k8s/deploy/dashboard` mounts read only. Locally, set +`OPEN_RL_CLIENT_RUNS_ROOT` to the directory above the client's `log_path`. +[Client metrics and logs](configuration.md#client-metrics-and-logs) says how a +directory is matched to a run. diff --git a/docs/designs/018-run-diagnostics.md b/docs/designs/018-run-diagnostics.md new file mode 100644 index 000000000..3e68e5e6e --- /dev/null +++ b/docs/designs/018-run-diagnostics.md @@ -0,0 +1,165 @@ +# 018: Run diagnostics + +**Status:** Proposed. + +A person or an agent asks one question of a run: why is it not behaving? OpenRL +cannot answer it today from the server: + +- Command records carry no step, and sample requests evict trainer ops. At the + default cap of 2,000 records, a run with eight samplers keeps under two steps. +- Where a worker ran is rebuilt from pods, device slices and Workloads, not + recorded. Samplers record nothing. +- A request lost between the Redis push and the trainer's pop looks the same + as a client that sends no work. +- Nothing records which worker holds a request, so a request held by a dead + worker looks like a slow one. + +This design records where each process ran, which step each command belongs +to, and which requests are outstanding. A read-only API under +`/api/v1/inspect` joins those records into facts. The dashboard UI and agents +use the same routes. + +## Records + +Every key is in the API server's Redis. Each record expires +`OPEN_RL_OPS_TTL_DAYS` (default 7) after its last write; `0` keeps it. + +| Record | Key | Writer | Holds | +|---|---|---|---| +| Command record | `open_rl:operations:` for trainer ops, `open_rl:operations::samples` for samples | Trainer, one per op. Sampler, one per failed sample only | `operation`, `status`, `error_type`, `started_at`, `at`, `elapsed_seconds`, `request_id`, `role`, `runtime_id`, `pod`, `pod_uid`, `node`, finite `metrics`, the names of non-finite ones in `nonfinite`, and `step` on trainer ops. The newest `OPEN_RL_OPS_SAMPLE_LIMIT` (2,000) of each list | +| Sample count and time | `open_rl:sampled:`, `open_rl:sampled_at:` | Sampler and torch trainer, on every sample | The run's samples sent, and when the last one finished. A vLLM sampler keeps one command record per group of samples it served, not one per sample. A torch trainer's samples keep theirs | +| Step counter and step times | `open_rl:steps:`, `open_rl:step_times:` | Trainer, on each successful `optim_step` | The run's step count, and the times of its last 64 steps | +| Request counters | `open_rl:ledger:`, one hash | API server (`enqueued`), trainer (`popped`, `answered`, `failed`) | Trainer requests that reached each point | +| Worker record | `open_rl:worker:`, one per process | Every trainer and sampler, rewritten every 5 s by a thread | Pod, pod UID, node, role, runtime, GPU UUIDs (`null` when NVML is unreadable), phase, recent GPU turns, outstanding requests by op and run with the oldest's age, and the settings env with secrets left out. A record not rewritten for 30 s belongs to a gone process | +| Session record | `open_rl:session_record:` | API server, on create, heartbeat and model attach | Created and last-seen times, the session's models, and its redacted metadata | +| SDK errors | `open_rl:sdk_errors:` | API server, from the SDK's telemetry | The last 200 error events, without credentials | +| Placement | `open_rl:placements:`, one record per Workload placement | API server, which lists the scheduler's Workloads every 30 s and before each release | Workload, role, pod, node, GPU UUIDs, placed and released times | +| Workload status | `status.gpuUUIDs`, `status.placedAt`, `status.releasedAt` | Scheduler, on bind and release | The source of placement records. The scheduler also exports `openrl_gpu_held{run_id, role, pod, node, UUID}` while a GPU is held and logs one `openrl.placement` line per change | + +Each event a writer records also prints one JSON line, so Cloud Logging keeps a +history longer than the Redis TTL. The line carries the Cloud Logging labels +`openrl_source=telemetry` and `openrl_run_id=` or `openrl_session_id=`, +and its `message` is `telemetry `. A run's kinds are `enqueued`, `popped`, +`answered` and `op` (a command record), all written through +`telemetry.record(run_id, kind, **fields)`. Other kinds are `placement`, +`session_created`, `model_attached` and `sdk_event`. An answered vLLM sample +and a session heartbeat print no line. Fields are redacted before they print. +No route reads these lines. + +A run's display name comes from the `name`, `run_name`, `recipe_name` and +`git_rev` keys of its `user_metadata`, kept in its model metadata. + +`openrl_gpu_held` joins DCGM series to runs. A time-sliced GPU has one series +per holder, so reduce it per UUID before the join: + +```promql +100 * DCGM_FI_PROF_GR_ENGINE_ACTIVE * on(UUID) group_left() max by (UUID) (openrl_gpu_held{run_id="$run", role="trainer"}) +``` + +Utilization comes from `DCGM_FI_PROF_GR_ENGINE_ACTIVE`, which averages the exporter's whole 10 s collection interval. +`DCGM_FI_DEV_GPU_UTIL` covers only about the last second of each interval, so a load with the same period aliases. On an +A100 running 5 s on and 5 s off, it read 88% while engine activity read 51%. The read API falls back to it where engine +activity is missing. + +## Routes + +`python -m server.dashboard` serves these routes. The API server serves them +too when `OPEN_RL_INSPECTION_API=true`. Every route is a `GET` under +`/api/v1/inspect`. + +| Route | Answers | +|---|---| +| `/runs?ended=` | Which run to open: every live run in one line (lifecycle, steps, seconds since its last op and step, median step time, unanswered requests and how many a dead worker holds, samples queued, failures, relaunches), longest since its last op first. Built from the counters, the last sample time and the last record of each list, never the command history | +| `/runs/{id}/summary` | What is wrong with this run: the run facts below, and GPU duty by role over the run's span | +| `/runs/{id}/steps?last=&fields=` | What each step did: trainer ops grouped by step, with each op's count, failures, seconds and metric spread, and in `client` the metrics the client wrote for the step, kept by the `fields` prefixes. `client_directory` names the client directory, or is null when none matched | +| `/runs/{id}/commands?since=&until=&role=` | The raw command records, newest first. A vLLM sampler's samples appear as the groups it served, with a count, and one record per failed sample | +| `/sessions/{id}/summary` | What a client did: its session record, whether it is live, its SDK errors, and a line per model over the session's span | +| `/inventory` | What exists now: runs with their lifecycle (`ended`, `waiting`, `placed`, `active`), nodes with their GPUs, the GPUs held now, pods with problems, and one error per source that could not be read | +| `/activity?run_id=&node_id=&since=&until=` | Who ran where and when: one lane per placement with its GPU turns and recorded op intervals. A sampler lane's samples are the groups it served | +| `/accelerators?run_id=&node_id=&since=&until=` | What each GPU is doing: node, slot, model, every placement holding it, utilization now and over the window, memory used of total | +| `/query?name=&run_id=&node_id=&since=&until=&points=` | A named PromQL query (`gpu_util`, `gpu_util_now`, `gpu_busy`, `gpu_memory_mib`, `gpu_memory_total_mib`, `vllm_*`) over a run's or node's GPUs and pods, or the fleet's GPUs | +| `/logs?run_id=&pod=&q=&severity=&since=&until=&cursor=` | The run's lines, newest first. From Cloud Logging, its worker pods by the scheduler's owner label and the lines an in-cluster client labels `openrl_run_id=` and `openrl_source=client`, without telemetry's own lines; a stderr line takes the level it prints. With `pod=client` and a client directory, its `logs.log`, whose lines carry no time, so the window does not apply and the cursor is a byte offset | + +Bounds are ISO-8601 or a span such as `1h`, at most seven days, and default to +the last 30 minutes. + +A run's client metrics and logs come from its client directory, the +tinker-cookbook's `log_path`, read where the server mounts it +(`OPEN_RL_CLIENT_RUNS_ROOT`, `/mnt/shared` by default). The client sends +nothing. `metrics.jsonl` holds one line per step with its `step`, read from the +offset the last read stopped at. `logs.log` is paged back from its end. A +directory is a run's when its `checkpoints.jsonl` names `tinker:///`, or +before the first save when its `config.json` was written within 60 s of the +run's creation for the run's base model. When the match is ambiguous, the run +has no client directory. `docs/configuration.md` gives the rules. +`timing_spans.jsonl` in the same directory is a later source for trace export. +Some runs keep client metrics in Redis under `open_rl:run_metrics:`; +`metrics.jsonl` wins a step both have. + +The `/runs/{id}` routes, `/activity` and `/logs` also take a session id in place +of a run id. A session with one run answers for that run. A session with several +runs answers 409 and names them. + +### Run facts + +`/runs/{id}/summary` returns each block only when it has content. An absent +block means nothing of its kind was recorded. + +| Block | Holds | +|---|---| +| `steps`, `step_seconds`, `since_last_step_s`, `last_op`, `last_metrics` | Optimizer steps, the median and last gap between them, and the newest trainer op | +| `since_last_op_s`, `trainer_idle_s` | For a live run, seconds since any op and since the trainer's last op | +| `goodput` | The share of the run's span spent training, the rest split into checkpoint, forward-only, rollout (a group of samples being served, or a recorded torch sample in flight) and client (nothing in flight) seconds | +| `requests` | Unanswered trainer requests with the holding pod and whether its worker still reports; the counters, with `lost = enqueued - popped - queued`; samples waiting in the sampler queue | +| `failures` | Failed trainer ops: the count, the 5 newest, and a log read around the newest | +| `relaunches` | Workers replaced in the same pod during the run, with a log read around the first. A clean exit, or a shared runtime's idle pod recreated under its name, is not one | +| `saves` | `save_state` calls, failures, the last success and the steps since | +| `optimizer` | Ops with non-finite metrics and the first of them, and the last 5 grad norms | +| `samplers` | Sends (the sample count), failures (the failed records), and the median time of the groups its samplers served | +| `config` | Workers whose echoed settings differ from what the run asked for, or `no echo` | +| `placement` | Placements recorded, and Workloads with no node yet | +| `incidents` | The 5 newest Kubernetes terminations and scheduling failures on the run's pods | +| `gpu_busy` | Mean GPU duty by role. A GPU that time-sliced roles shared counts under the roles joined, such as `sampler+trainer` | + +A log read is a route and its arguments, such as +`{"read": "logs", "args": {"run_id": "…", "pod": "…", "since": "…", "until": "…"}}`, +so an agent follows a failure to its traceback in one call. + +## Rules + +- **Facts, not verdicts.** No route says whether a run is healthy. There are + no thresholds; the reader draws the conclusion. +- **Telemetry never raises or blocks.** Every hook suppresses its own errors. + Every store write runs in the background and is dropped after 5 s. An op + returns or raises exactly as it would without telemetry, even when the store + raises, hangs or is missing. `tests/test_telemetry_faults.py` checks this. +- **Reads are bounded and shared.** The fleet snapshot is cached for 5 s and a + run's command records for 10 s. A stale entry is refreshed by the next + caller, and every caller gets the last good result until the refresh lands, + unless that result is older than both 60 s and six windows. A failed read is + retried by the next caller. The 32 most recently used entries are kept. +- **Logs stay in their run.** A log read takes only pods with the run's owner + label, so a run ID cannot open another run's logs. +- **The client API does not list the fleet.** The API server mounts the routes + only with `OPEN_RL_INSPECTION_API=true`. + +## Known gaps + +- Workers return a NaN loss or grad norm as 0 and infinity as ±9999 + (`sanitize_float`), so those read as numbers in command records. Other + non-finite metrics are named in `nonfinite`. +- A local stack has no Cloud Logging and no Workloads, so its runs have no + logs or placement blocks. +- Runs recorded before step stamps, step times or the failure counter leave + those fields null, and their steps are derived from the order of their ops. +- Sampler latency is the time a group of samples took, not each request's. Runs + recorded before group records, which kept neither, have no sampler latency, + rollout split or sample bars. vLLM's per-pod charts stay on `/query`. +- Client metrics and logs need the client's directory on a volume the server + reads. A run whose `log_path` is not on the shared volume, and a client on a + laptop, have none. Runs started together on one base model match no directory + by time, so each has none until its first checkpoint. +- Facts are computed on request. Alerts would need a timer and thresholds. +- Not recorded yet: cgroup OOM kills and peak memory in the worker record, NVML + utilization for GPU facts without Prometheus, and the request's holder + incarnation in the counters. diff --git a/k8s/deploy/dashboard/dashboard.yaml b/k8s/deploy/dashboard/dashboard.yaml new file mode 100644 index 000000000..3626b26aa --- /dev/null +++ b/k8s/deploy/dashboard/dashboard.yaml @@ -0,0 +1,68 @@ +# The dashboard as its own process (python -m server.dashboard), so it can be +# restarted, scaled, or broken without touching the API server. It reuses the +# slim API-server image: same dependencies, different entrypoint. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: open-rl-dashboard + labels: + app: open-rl-dashboard +spec: + replicas: 1 + selector: + matchLabels: + app: open-rl-dashboard + template: + metadata: + labels: + app: open-rl-dashboard + spec: + serviceAccountName: open-rl-sa + containers: + - name: dashboard + image: ghcr.io/gke-labs/open-rl/api-server:latest + command: ["uv", "run", "python", "-m", "server.dashboard", "--host", "0.0.0.0", "--port", "8000"] + ports: + - name: http + containerPort: 8000 + env: + - name: REDIS_URL + value: redis://redis-service:6379 + - name: OPEN_RL_WORKER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # GKE metrics use Cloud Monitoring's PromQL API (monitoring.viewer on + # the workload identity). Set OPEN_RL_PROMETHEUS_URL for another backend. + # A long run's activity keeps the process busy for seconds; the default + # one-second timeout took the pod out of its Service meanwhile. + readinessProbe: + httpGet: + path: /openapi.json + port: 8000 + timeoutSeconds: 5 + resources: + requests: + cpu: 250m + memory: 512Mi + # Each run's client directory, its tinker-cookbook log_path, holds its metrics and logs. + volumeMounts: + - name: shared-storage + mountPath: /mnt/shared + readOnly: true + volumes: + - name: shared-storage + persistentVolumeClaim: + claimName: open-rl-shared-pvc + readOnly: true +--- +apiVersion: v1 +kind: Service +metadata: + name: open-rl-dashboard +spec: + selector: + app: open-rl-dashboard + ports: + - port: 8000 + targetPort: 8000 diff --git a/k8s/deploy/dashboard/kustomization.yaml b/k8s/deploy/dashboard/kustomization.yaml new file mode 100644 index 000000000..266472852 --- /dev/null +++ b/k8s/deploy/dashboard/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: openrl-system +resources: + - rbac.yaml + - dashboard.yaml +images: + - name: ghcr.io/gke-labs/open-rl/api-server + newTag: latest diff --git a/k8s/deploy/dashboard/rbac.yaml b/k8s/deploy/dashboard/rbac.yaml new file mode 100644 index 000000000..79db03a2e --- /dev/null +++ b/k8s/deploy/dashboard/rbac.yaml @@ -0,0 +1,48 @@ +# The dashboard lists pods and Workloads in the namespace and nodes cluster-wide. It runs as the API server's account, +# so an API server with OPEN_RL_INSPECTION_API=true needs this applied too; base grants no node reads. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: open-rl-dashboard +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["list"] +- apiGroups: ["openrl.io"] + resources: ["workloads"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: open-rl-dashboard +subjects: +- kind: ServiceAccount + name: open-rl-sa +roleRef: + kind: Role + name: open-rl-dashboard + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: open-rl-dashboard +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: open-rl-dashboard +subjects: +- kind: ServiceAccount + name: open-rl-sa + # A cluster binding's subject needs its namespace; kustomize does not fill it in. + namespace: openrl-system +roleRef: + kind: ClusterRole + name: open-rl-dashboard + apiGroup: rbac.authorization.k8s.io diff --git a/k8s/deploy/distributed-fft-timeslice/10-dcgm-monitoring.yaml b/k8s/deploy/distributed-fft-timeslice/10-dcgm-monitoring.yaml index 27a536a92..bfd0ebbda 100644 --- a/k8s/deploy/distributed-fft-timeslice/10-dcgm-monitoring.yaml +++ b/k8s/deploy/distributed-fft-timeslice/10-dcgm-monitoring.yaml @@ -30,6 +30,10 @@ data: DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB). # FieldID 252 DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB). + + # Profiling metrics, averaged over the collection interval,, + # FieldID 1001 + DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active. --- apiVersion: apps/v1 kind: DaemonSet @@ -78,6 +82,9 @@ spec: value: ":9400" - name: DISABLE_STARTUP_VALIDATE value: "true" + # A reading every 10 s, as often as it is scraped; the 30 s default left two of every three scrapes repeating it. + - name: DCGM_EXPORTER_INTERVAL + value: "10000" - name: LD_LIBRARY_PATH value: "/usr/local/nvidia/lib64" resources: diff --git a/pyproject.toml b/pyproject.toml index 804694c58..299fe3bea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,9 @@ dev = [ where = ["src"] include = ["server*", "training*", "accel_timeslicer*"] +[tool.setuptools.package-data] +"server.dashboard" = ["static/*"] + [tool.uv] override-dependencies = [ # vLLM still carries an older transformers constraint; keep the project on diff --git a/scheduler/controller/api/v1alpha1/workload_types.go b/scheduler/controller/api/v1alpha1/workload_types.go index 57c3d8cbf..9496116a9 100644 --- a/scheduler/controller/api/v1alpha1/workload_types.go +++ b/scheduler/controller/api/v1alpha1/workload_types.go @@ -171,6 +171,20 @@ type WorkloadStatus struct { // the node is genuinely unknown. NodeName string `json:"nodeName,omitempty"` + // GPUUUIDs are the UUIDs of the claim's allocated devices, which GPU + // metrics are keyed by. + // +optional + GPUUUIDs []string `json:"gpuUUIDs,omitempty"` + + // PlacedAt is when the controller observed the allocation and set NodeName. + // +optional + PlacedAt *metav1.Time `json:"placedAt,omitempty"` + + // ReleasedAt is when the worker stopped holding its devices. Cleared by + // the next placement. + // +optional + ReleasedAt *metav1.Time `json:"releasedAt,omitempty"` + // Reason is the most recent human-readable explanation of Phase. Reason string `json:"reason,omitempty"` diff --git a/scheduler/controller/api/v1alpha1/zz_generated.deepcopy.go b/scheduler/controller/api/v1alpha1/zz_generated.deepcopy.go index ef6ccd2c7..e5c84407a 100644 --- a/scheduler/controller/api/v1alpha1/zz_generated.deepcopy.go +++ b/scheduler/controller/api/v1alpha1/zz_generated.deepcopy.go @@ -200,6 +200,19 @@ func (in *WorkloadSpec) DeepCopy() *WorkloadSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkloadStatus) DeepCopyInto(out *WorkloadStatus) { *out = *in + if in.GPUUUIDs != nil { + in, out := &in.GPUUUIDs, &out.GPUUUIDs + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PlacedAt != nil { + in, out := &in.PlacedAt, &out.PlacedAt + *out = (*in).DeepCopy() + } + if in.ReleasedAt != nil { + in, out := &in.ReleasedAt, &out.ReleasedAt + *out = (*in).DeepCopy() + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) diff --git a/scheduler/controller/go.mod b/scheduler/controller/go.mod index 1ba836367..4b5328e76 100644 --- a/scheduler/controller/go.mod +++ b/scheduler/controller/go.mod @@ -3,6 +3,7 @@ module github.com/gke-labs/open-rl/scheduler/controller go 1.26.0 require ( + github.com/prometheus/client_golang v1.23.2 k8s.io/api v0.36.3 k8s.io/apimachinery v0.36.3 k8s.io/client-go v0.36.3 @@ -32,7 +33,6 @@ require ( github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect diff --git a/scheduler/controller/internal/controller/fleet.go b/scheduler/controller/internal/controller/fleet.go index f805aa46b..0fd79919a 100644 --- a/scheduler/controller/internal/controller/fleet.go +++ b/scheduler/controller/internal/controller/fleet.go @@ -3,11 +3,14 @@ package controller import ( "context" "fmt" + "path" "sort" + "strings" corev1 "k8s.io/api/core/v1" resourcev1 "k8s.io/api/resource/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -250,6 +253,43 @@ func latestCompletePools(ctx context.Context, slices []resourcev1.ResourceSlice, return keep } +// gpuUUIDs resolves the devices allocated to a claim to the UUIDs their +// ResourceSlices publish, in allocation order. A device without a UUID +// attribute is left out; an empty result is retried on a later reconcile. +func (r *WorkloadReconciler) gpuUUIDs(ctx context.Context, claimName string) ([]string, error) { + var claim resourcev1.ResourceClaim + if err := r.Get(ctx, types.NamespacedName{Namespace: r.Namespace, Name: claimName}, &claim); err != nil { + return nil, client.IgnoreNotFound(err) + } + if claim.Status.Allocation == nil { + return nil, nil + } + var slices resourcev1.ResourceSliceList + if err := r.List(ctx, &slices); err != nil { + return nil, fmt.Errorf("list resourceslices: %w", err) + } + type deviceKey struct{ driver, pool, device string } + published := map[deviceKey]string{} + for _, i := range latestCompletePools(ctx, slices.Items, r.DeviceDriver) { + spec := slices.Items[i].Spec + for _, device := range spec.Devices { + for name, attr := range device.Attributes { + // NVIDIA's driver publishes "uuid"; match any domain and case. + if attr.StringValue != nil && strings.EqualFold(path.Base(string(name)), "uuid") { + published[deviceKey{spec.Driver, spec.Pool.Name, device.Name}] = *attr.StringValue + } + } + } + } + var uuids []string + for _, result := range claim.Status.Allocation.Devices.Results { + if uuid, ok := published[deviceKey{result.Driver, result.Pool, result.Device}]; ok { + uuids = append(uuids, uuid) + } + } + return uuids, nil +} + // claimFrom reads a managed claim back into the fleet: identity from // metadata, shape from DRA's allocation. An unallocated claim has no shape // yet -- the tiers were alternatives, and which one holds is DRA's answer. diff --git a/scheduler/controller/internal/controller/metrics.go b/scheduler/controller/internal/controller/metrics.go new file mode 100644 index 000000000..57ef4c75a --- /dev/null +++ b/scheduler/controller/internal/controller/metrics.go @@ -0,0 +1,50 @@ +package controller + +import ( + "context" + "time" + + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/client" + + openrlv1alpha1 "github.com/gke-labs/open-rl/scheduler/controller/api/v1alpha1" +) + +// The UUID label is spelled as DCGM spells it, so GPU metrics join on it. +var gpuHeldDesc = prometheus.NewDesc( + "openrl_gpu_held", + "1 for each GPU a placed workload holds. Present from placement to release, so its presence over time is the hold interval.", + []string{"run_id", "role", "pod", "node", "UUID"}, nil, +) + +// gpuHeldCollector reads workload status at scrape time, so a released or +// deleted workload's series disappears with it. +type gpuHeldCollector struct { + reader client.Reader + namespace string +} + +func (c gpuHeldCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- gpuHeldDesc +} + +func (c gpuHeldCollector) Collect(ch chan<- prometheus.Metric) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var workers openrlv1alpha1.WorkloadList + if err := c.reader.List(ctx, &workers, client.InNamespace(c.namespace)); err != nil { + ch <- prometheus.NewInvalidMetric(gpuHeldDesc, err) + return + } + for i := range workers.Items { + worker := &workers.Items[i] + status := worker.Status + if status.NodeName == "" || status.ReleasedAt != nil { + continue + } + for _, uuid := range status.GPUUUIDs { + ch <- prometheus.MustNewConstMetric(gpuHeldDesc, prometheus.GaugeValue, 1, + worker.Spec.ModelID, string(worker.Spec.Role), status.PodName, status.NodeName, uuid) + } + } +} diff --git a/scheduler/controller/internal/controller/metrics_test.go b/scheduler/controller/internal/controller/metrics_test.go new file mode 100644 index 000000000..91eb21b8c --- /dev/null +++ b/scheduler/controller/internal/controller/metrics_test.go @@ -0,0 +1,112 @@ +package controller + +import ( + "context" + "fmt" + "slices" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + corev1 "k8s.io/api/core/v1" + resourcev1 "k8s.io/api/resource/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + openrlv1alpha1 "github.com/gke-labs/open-rl/scheduler/controller/api/v1alpha1" +) + +// heldGPUs scrapes openrl_gpu_held as the metrics endpoint does, one +// "run role pod node UUID=value" string per series. +func heldGPUs(t *testing.T, r *WorkloadReconciler) []string { + t.Helper() + registry := prometheus.NewPedanticRegistry() + registry.MustRegister(gpuHeldCollector{reader: r.Client, namespace: testNamespace}) + families, err := registry.Gather() + must(t, err) + var series []string + for _, family := range families { + for _, metric := range family.GetMetric() { + l := map[string]string{} + for _, pair := range metric.GetLabel() { + l[pair.GetName()] = pair.GetValue() + } + series = append(series, fmt.Sprintf("%s %s %s %s %s=%v", l["run_id"], l["role"], l["pod"], l["node"], l["UUID"], metric.GetGauge().GetValue())) + } + } + return series +} + +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} + +// placedWorker returns a reconciler whose worker w-a holds gpu-0, and its claim. +// The observed allocation stamps the GPUs and the time, and the metric reports +// the hold. +func placedWorker(t *testing.T) (*WorkloadReconciler, string) { + t.Helper() + r := newReconciler(t, append(enabledNode(), trainerWorker("w-a", "model-a"))...) + settle(t, r, "w-a") + claimName := claimOf(t, r, "w-a") + allocateClaim(t, r, claimName) + runReconcile(t, r, "w-a") + placed := getWorker(t, r, "w-a").Status + if placed.NodeName != testNode || !slices.Equal(placed.GPUUUIDs, []string{"GPU-gpu-0"}) || placed.PlacedAt == nil || placed.ReleasedAt != nil { + t.Fatalf("status = node %q uuids %v placedAt %v releasedAt %v, want node-a, GPU-gpu-0, placed, not released", + placed.NodeName, placed.GPUUUIDs, placed.PlacedAt, placed.ReleasedAt) + } + if got, want := heldGPUs(t, r), []string{"model-a trainer orw-w-a node-a GPU-gpu-0=1"}; !slices.Equal(got, want) { + t.Fatalf("series = %v, want %v", got, want) + } + return r, claimName +} + +func expectReleased(t *testing.T, r *WorkloadReconciler) { + t.Helper() + status := getWorker(t, r, "w-a").Status + if status.Phase != openrlv1alpha1.PhasePending || status.ReleasedAt == nil || status.NodeName != "" || len(status.GPUUUIDs) != 0 { + t.Fatalf("status = phase %q node %q uuids %v releasedAt %v, want Pending, released, no node or GPUs", + status.Phase, status.NodeName, status.GPUUUIDs, status.ReleasedAt) + } + if got := heldGPUs(t, r); len(got) != 0 { + t.Fatalf("series after release = %v, want none", got) + } +} + +// A placed worker stops holding its GPUs as soon as the controller sees its +// claim abandoned or gone, whether re-placement waits on the old claim's +// deletion or finds no capacity. +func TestLostClaimReleasesBeforeRePlacement(t *testing.T) { + ctx := context.Background() + cases := map[string]func(t *testing.T, r *WorkloadReconciler, claimName string){ + "wedged claim abandoned": func(t *testing.T, r *WorkloadReconciler, _ string) { + markUnschedulable(t, r, "w-a", time.Now().Add(-2*wedgeGracePeriod)) + }, + "old claim still terminating": func(t *testing.T, r *WorkloadReconciler, claimName string) { + var claim resourcev1.ResourceClaim + must(t, r.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: claimName}, &claim)) + claim.Finalizers = append(claim.Finalizers, "test.openrl.io/hold") + must(t, r.Update(ctx, &claim)) + must(t, r.Delete(ctx, &claim)) + }, + "no capacity left": func(t *testing.T, r *WorkloadReconciler, claimName string) { + must(t, r.Delete(ctx, &resourcev1.ResourceClaim{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: claimName}})) + var node corev1.Node + must(t, r.Get(ctx, types.NamespacedName{Name: testNode}, &node)) + node.Labels[NodeLabelEnabled] = "false" + must(t, r.Update(ctx, &node)) + }, + } + for name, loseClaim := range cases { + t.Run(name, func(t *testing.T) { + r, claimName := placedWorker(t) + loseClaim(t, r, claimName) + runReconcile(t, r, "w-a") + expectReleased(t, r) + }) + } +} diff --git a/scheduler/controller/internal/controller/workload_controller.go b/scheduler/controller/internal/controller/workload_controller.go index d8dfe8856..319d29fcb 100644 --- a/scheduler/controller/internal/controller/workload_controller.go +++ b/scheduler/controller/internal/controller/workload_controller.go @@ -23,6 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -201,6 +202,16 @@ func (r *WorkloadReconciler) repairPlacementAssignment(ctx context.Context, scop return placementPhaseResult{}, err } scope.claimName = "" + // Record the loss now. A re-cut reuses the claim's name, and the + // later Pending or Failed patches touch neither claim nor node, + // so this is the only transition that releases the placement. + if err := r.patchStatus(ctx, worker, func(status *openrlv1alpha1.WorkloadStatus) { + status.Phase = openrlv1alpha1.PhasePending + status.ClaimName, status.AssignmentID, status.NodeName = "", "", "" + status.Reason = "ReplacingLostClaim: the assigned claim no longer exists" + }); err != nil { + return placementPhaseResult{}, err + } } } // One seat per worker. Any other seat under this name is stale, left by @@ -512,6 +523,14 @@ func (r *WorkloadReconciler) teardown(ctx context.Context, worker *openrlv1alpha return ctrl.Result{}, err } } + if err := r.patchStatus(ctx, worker, func(status *openrlv1alpha1.WorkloadStatus) { + if status.PlacedAt != nil && status.ReleasedAt == nil { + now := metav1.Now() + status.ReleasedAt = &now + } + }); err != nil { + return ctrl.Result{}, err + } controllerutil.RemoveFinalizer(worker, workerFinalizer) return ctrl.Result{}, r.Update(ctx, worker) } @@ -694,9 +713,19 @@ func (r *WorkloadReconciler) reportPod(ctx context.Context, worker *openrlv1alph node = claim.Node } } + var uuids []string + if node != "" && len(worker.Status.GPUUUIDs) == 0 { + var err error + if uuids, err = r.gpuUUIDs(ctx, claimName); err != nil { + return err + } + } return r.patchStatus(ctx, worker, func(status *openrlv1alpha1.WorkloadStatus) { status.Phase, status.ClaimName, status.PodName, status.NodeName, status.Reason = phase, claimName, podName, node, reason + if len(uuids) > 0 { + status.GPUUUIDs = uuids + } // The footprint is DRA's answer, recordable only once the // allocation is observed: which tier held, and what each device // carries under it. @@ -715,6 +744,7 @@ func (r *WorkloadReconciler) reportPod(ctx context.Context, worker *openrlv1alph func (r *WorkloadReconciler) patchStatus(ctx context.Context, worker *openrlv1alpha1.Workload, mutate func(*openrlv1alpha1.WorkloadStatus)) error { before := worker.Status.DeepCopy() mutate(&worker.Status) + trackPlacement(before, &worker.Status, metav1.Now()) worker.Status.ObservedGeneration = worker.Generation // Skip no-op writes: they would re-trigger the watch forever. if apiequality.Semantic.DeepEqual(before, &worker.Status) { @@ -727,9 +757,43 @@ func (r *WorkloadReconciler) patchStatus(ctx context.Context, worker *openrlv1al } return fmt.Errorf("patch status of %s: %w", worker.Name, err) } + logPlacement(ctx, worker, before) return nil } +// trackPlacement stamps the placement interval on every status write. A worker +// is placed when NodeName is set and released when NodeName clears or its claim +// changes. Teardown releases it explicitly, since nothing else changes there. +func trackPlacement(before, after *openrlv1alpha1.WorkloadStatus, now metav1.Time) { + held := before.NodeName != "" && before.PlacedAt != nil && before.ReleasedAt == nil + if held && (after.NodeName == "" || after.ClaimName != before.ClaimName) { + after.NodeName, after.GPUUUIDs, after.ReleasedAt = "", nil, &now + return + } + if after.NodeName != "" && (after.NodeName != before.NodeName || after.PlacedAt == nil) { + after.PlacedAt, after.ReleasedAt = &now, nil + } +} + +// logPlacement writes one JSON line per persisted bind or release. GKE turns +// it into a structured entry the diagnostics read by kind. +func logPlacement(ctx context.Context, worker *openrlv1alpha1.Workload, before *openrlv1alpha1.WorkloadStatus) { + after := &worker.Status + event, held, at := "placed", after, after.PlacedAt + switch { + case after.PlacedAt != nil && !after.PlacedAt.Equal(before.PlacedAt): + case after.ReleasedAt != nil && before.ReleasedAt == nil: + // A release may clear the node and devices, so report what was held. + event, held, at = "released", before, after.ReleasedAt + default: + return + } + log.FromContext(ctx).Info("placement "+event, + "kind", "openrl.placement", "event", event, "run_id", worker.Spec.ModelID, "role", worker.Spec.Role, + "workload", worker.Name, "pod", held.PodName, "node", held.NodeName, "uuids", held.GPUUUIDs, + "time", at.UTC().Format(time.RFC3339)) +} + func (r *WorkloadReconciler) markPending(ctx context.Context, worker *openrlv1alpha1.Workload, reason string) error { return r.patchStatus(ctx, worker, func(status *openrlv1alpha1.WorkloadStatus) { status.Phase, status.Reason = openrlv1alpha1.PhasePending, reason @@ -792,6 +856,9 @@ func managedClaims() predicate.Predicate { // SetupWithManager registers the reconciler. func (r *WorkloadReconciler) SetupWithManager(mgr ctrl.Manager) error { r.reader = mgr.GetAPIReader() + if err := metrics.Registry.Register(gpuHeldCollector{reader: mgr.GetClient(), namespace: r.Namespace}); err != nil { + return err + } // Capacity changes are fleet-wide: a freed claim might unblock any // pending worker, so these events wake every worker rather than one. diff --git a/scheduler/controller/internal/controller/workload_controller_test.go b/scheduler/controller/internal/controller/workload_controller_test.go index a9658d6db..c1d044fb5 100644 --- a/scheduler/controller/internal/controller/workload_controller_test.go +++ b/scheduler/controller/internal/controller/workload_controller_test.go @@ -79,10 +79,12 @@ func enabledNode() []client.Object { nodeName := testNode product := "NVIDIA RTX PRO 6000 Blackwell" device := func(name string) resourcev1.Device { + uuid := "GPU-" + name return resourcev1.Device{ Name: name, Attributes: map[resourcev1.QualifiedName]resourcev1.DeviceAttribute{ "productName": {StringValue: &product}, + "uuid": {StringValue: &uuid}, }, Capacity: map[resourcev1.QualifiedName]resourcev1.DeviceCapacity{ "memory": {Value: resource.MustParse("96Gi")}, diff --git a/scheduler/deploy/base/00-workload-crd.yaml b/scheduler/deploy/base/00-workload-crd.yaml index 2042ce6bb..9eb73f66d 100644 --- a/scheduler/deploy/base/00-workload-crd.yaml +++ b/scheduler/deploy/base/00-workload-crd.yaml @@ -8773,6 +8773,13 @@ spec: the pools registered. format: int32 type: integer + gpuUUIDs: + description: |- + GPUUUIDs are the UUIDs of the claim's allocated devices, which GPU + metrics are keyed by. + items: + type: string + type: array memoryPerDevice: description: |- MemoryPerDevice is what each of those devices must provide, i.e. Memory @@ -8797,6 +8804,11 @@ spec: - Succeeded - Failed type: string + placedAt: + description: PlacedAt is when the controller observed the allocation + and set NodeName. + format: date-time + type: string podName: description: PodName is the worker pod, once created. type: string @@ -8804,6 +8816,12 @@ spec: description: Reason is the most recent human-readable explanation of Phase. type: string + releasedAt: + description: |- + ReleasedAt is when the worker stopped holding its devices. Cleared by + the next placement. + format: date-time + type: string type: object required: - spec diff --git a/src/server/Dockerfile.api_server b/src/server/Dockerfile.api_server index 6cd6207ce..b7eac5b8d 100644 --- a/src/server/Dockerfile.api_server +++ b/src/server/Dockerfile.api_server @@ -1,3 +1,11 @@ +FROM node:22-bookworm-slim AS dashboard-build +WORKDIR /build +COPY dev/dashboard/package*.json dev/dashboard/ +RUN npm --prefix dev/dashboard ci +COPY dev/dashboard/build.mjs dev/dashboard/ +COPY src/server/dashboard/ui src/server/dashboard/ui +RUN npm --prefix dev/dashboard run build + # Use lightweight standard Python image (No CUDA) FROM python:3.12-slim @@ -35,6 +43,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # Copy source code COPY . . +COPY --from=dashboard-build /build/src/server/dashboard/static/app.js src/server/dashboard/static/app.js # Expose source code via PYTHONPATH instead of modifying .venv after COPY . . ENV PYTHONPATH="/app/src:/app:$PYTHONPATH" diff --git a/src/server/api_server.py b/src/server/api_server.py index b8369afc4..9bf612759 100644 --- a/src/server/api_server.py +++ b/src/server/api_server.py @@ -30,12 +30,14 @@ from server.model_metadata import ( TrainingModelMetadata, WeightSyncConfig, + display_metadata, get_model_metadata, persist_model_metadata, ) from server.sampling_store import SamplingStore from server.session_registry import SessionRegistry from server.store import RedisStateStore, get_state_store, get_store +from server.telemetry import ingest_sdk_events, record, record_placements, write from server.vllm_options import split_stop from server.worker_manager import LocalWorkerManager, WorkerManager, create_worker_manager, default_metadata, owner_of from training import commands @@ -64,16 +66,34 @@ async def bind_session(session_id: str | None, model_id: str) -> None: if meta is not None and session_id not in meta.session_ids: meta.session_ids.append(session_id) await persist_model_metadata(state, model_id, meta) + # A sampling session on a bare base model names no run. + if meta is not None: + session_registry.note(session_id, model_id=model_id) # On the owner, for the reaper: workers stay while any attached session lives. if worker_manager is not None: await session_registry.attach(session_id, owner) +async def sweep_placements() -> None: + """Placement records from the scheduler's Workloads, also taken before a release deletes them. A failed sweep is retried.""" + try: + workloads = await asyncio.wait_for(asyncio.to_thread(worker_manager.workloads), timeout=5) + except Exception as exc: + print(f"[API_SERVER] Placement sweep skipped: {exc!r}") + return + try: + terminations = await asyncio.wait_for(asyncio.to_thread(worker_manager.pod_terminations), timeout=5) + except Exception: # placements stand without their pods' exit reasons + terminations = {} + write(record_placements(state, workloads, terminations), timeout=SESSION_REAP_INTERVAL_SEC) + + async def reap_owner(owner: str) -> None: async with owner_locks[owner]: if await session_registry.in_use(owner): return print(f"[API_SERVER] No live session uses {owner}; tearing its workers down") + await sweep_placements() for model in await asyncio.to_thread(worker_manager.release_owner, owner): await state.delete_values(f"open_rl:sampler_ready:{model}", f"open_rl:trainer_ready:{model}") await session_registry.forget(owner) @@ -485,7 +505,8 @@ async def _extract_and_persist_model_metadata( lora_config = req.lora_config.model_dump() # The model's own user_metadata wins over what the session was opened with. - settings = resolve_settings(req.user_metadata or {}, await session_registry.user_metadata(req.session_id)) + session_meta = await session_registry.user_metadata(req.session_id) + settings = resolve_settings(req.user_metadata or {}, session_meta) if checkpoint_kind is not None and settings.fine_tuning_type not in (None, checkpoint_kind): raise ValueError(f"{SETTINGS_PREFIX}fine_tuning_type={settings.fine_tuning_type} does not match the {checkpoint_kind} checkpoint") fine_tuning_type = settings.fine_tuning_type or checkpoint_kind or "lora" @@ -529,6 +550,7 @@ async def _extract_and_persist_model_metadata( trainer_gpus=settings.trainer_gpus, trainer_cp=settings.trainer_cp, recipe=settings.recipe, + **display_metadata({**session_meta, **(req.user_metadata or {})}), ) await persist_model_metadata(state, model_id, meta_obj) @@ -561,9 +583,8 @@ async def enqueue(command: Command) -> str: active_set_id = await _resolve_active_set_id(command.model_id) await store.put_request(commands.wire(command.model_copy(update={"trace_context": carrier})), active_set_id=active_set_id) - # One line per training request so a request that never reaches a worker can - # be traced end to end (the workers log the same id when they pop it). - print(f"[API_SERVER] enqueued op={command.op} request_id={command.request_id} model_id={command.model_id} active_set={active_set_id}") + # Its log line lets a request that never reaches a worker be traced by its id, which the worker logs when it pops it. + record(command.model_id, "enqueued", request_id=command.request_id, op=command.op, active_set=active_set_id) return command.request_id @@ -652,6 +673,7 @@ async def reap_dead_sessions(): while True: await asyncio.sleep(SESSION_REAP_INTERVAL_SEC) # Nothing in a sweep may end the loop. A failed call is retried next sweep. + await sweep_placements() try: owners = await session_registry.owners() except Exception: @@ -735,6 +757,11 @@ def openapi_with_settings() -> dict[str, Any]: app.openapi = openapi_with_settings +# Fleet inspection belongs on the operator dashboard, not the model-capability API. +if os.getenv("OPEN_RL_INSPECTION_API", "").lower() == "true": + from server.dashboard.router import router as dashboard_router + + app.include_router(dashboard_router) FastAPIInstrumentor.instrument_app(app, excluded_urls="/api/v1/retrieve_future,/api/v1/session_heartbeat") @@ -921,6 +948,7 @@ async def delete_model(req: ModelRequest): print(f"[API_SERVER] Requesting shutdown of workers for model {model_id}...") await store.put_request(commands.wire(commands.Shutdown(model_id=model_id)), active_set_id=await _resolve_active_set_id(model_id)) await store.put_sampling_request({"request_id": "SHUTDOWN_SENTINEL", "model_id": model_id}) + await sweep_placements() await asyncio.to_thread(worker_manager.release, model_id) now = time.time() meta.status = "completed" @@ -1319,6 +1347,8 @@ async def asample(req: AsampleRequest): "weights_path": weights_path, "include_prompt_logprobs": req.prompt_logprobs, "model_id": queue_id, + # The queue may be a shared runtime's; the run is the model the client samples. + "logical_run_id": lookup_id, "trace_context": carrier, } @@ -1356,5 +1386,6 @@ async def list_adapters(): @app.post("/api/v1/telemetry") -async def telemetry(_: dict): +async def telemetry(body: dict): + ingest_sdk_events(state, body) return {"status": "accepted"} diff --git a/src/server/dashboard/__init__.py b/src/server/dashboard/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/server/dashboard/__main__.py b/src/server/dashboard/__main__.py new file mode 100644 index 000000000..13d0eed27 --- /dev/null +++ b/src/server/dashboard/__main__.py @@ -0,0 +1,23 @@ +"""The dashboard as its own process, without the client API's queues or worker manager: `python -m server.dashboard`.""" + +import argparse + +import uvicorn +from fastapi import FastAPI +from fastapi.middleware.gzip import GZipMiddleware +from fastapi.responses import RedirectResponse + +from server.dashboard.router import router + +app = FastAPI(title="Open-RL Inspection API") +app.add_middleware(GZipMiddleware, minimum_size=4096, compresslevel=1) # a port-forward carries large JSON slowly +app.include_router(router) +app.get("/", include_in_schema=False)(lambda: RedirectResponse("/dashboard")) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + # Flags only: Kubernetes injects OPEN_RL_DASHBOARD_PORT=tcp://... for a Service named open-rl-dashboard. + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=8080) + args = parser.parse_args() + uvicorn.run(app, host=args.host, port=args.port) diff --git a/src/server/dashboard/client_runs.py b/src/server/dashboard/client_runs.py new file mode 100644 index 000000000..35e37e66c --- /dev/null +++ b/src/server/dashboard/client_runs.py @@ -0,0 +1,132 @@ +"""A run's client directory: the tinker-cookbook's log_path on a volume the server also mounts, where the client writes +config.json, metrics.jsonl, checkpoints.jsonl and logs.log. The client does nothing for the server. Every read is +blocking, so callers run it in a thread.""" + +import json +import os +import re + +from server.telemetry import finite + +DEPTH = 3 # the root's run directories, such as /mnt/shared// or one level deeper +MATCH_SECONDS = 60 # config.json is written just before the client creates its model +SCAN_BYTES = 4 << 20 # of logs.log per page +MESSAGE_CHARS = 16384 +TINKER_PATH = re.compile(r'tinker://([^/"]+)/') +# The cookbook's file format, "%(name)s:%(lineno)d [%(levelname)s] %(message)s"; other lines continue the record above. +HEADER = re.compile(r"\S+:\d+ \[(DEBUG|INFO|WARNING|ERROR|CRITICAL)\] ") +listings: dict[str, tuple[int, list[str], bool]] = {} # directory -> (mtime, subdirectories, is a run directory) +parsed: dict[str, tuple[tuple[int, int], object]] = {} # file -> ((mtime, size), parsed contents) +progress: dict[str, tuple[int, int, dict[int, dict[str, float]]]] = {} # metrics.jsonl -> (inode, bytes read, metrics by step) + + +def root() -> str: + return os.getenv("OPEN_RL_CLIENT_RUNS_ROOT", "/mnt/shared") + + +def directories(path: str | None = None, depth: int = DEPTH) -> list[str]: + """Every directory under the root, to DEPTH, that holds config.json or checkpoints.jsonl. A run directory is not + descended into, and a directory is listed again only when its mtime changes.""" + path = path or root() + try: + mtime = os.stat(path).st_mtime_ns + if (listing := listings.get(path)) is None or listing[0] != mtime: + with os.scandir(path) as scan: + entries = list(scan) + run = any(e.name in ("config.json", "checkpoints.jsonl") for e in entries) + below = [] if run else [e.path for e in entries if not e.name.startswith(".") and e.name != "lost+found" and e.is_dir(follow_symlinks=False)] + listing = listings[path] = (mtime, below, run) + except OSError: # no root here, or a directory removed while it was scanned + return [] + if listing[2]: + return [path] + return [found for below in listing[1] for found in directories(below, depth - 1)] if depth > 0 else [] + + +def read(path: str, parse) -> tuple[float, object] | None: + """The file's mtime and parsed contents, read again only when its mtime or size changes. None when it is absent, as + checkpoints.jsonl is until the first save, or is not what the cookbook writes.""" + try: + stat = os.stat(path) + if (entry := parsed.get(path)) is None or entry[0] != (stat.st_mtime_ns, stat.st_size): + with open(path, encoding="utf-8", errors="replace") as f: + entry = parsed[path] = ((stat.st_mtime_ns, stat.st_size), parse(f.read())) + except (OSError, ValueError): + return None + return entry[0][0] / 1e9, entry[1] + + +def started(directory: str, run: dict) -> bool: + """Whether config.json was written within MATCH_SECONDS of the run's creation, for its base model.""" + if not (found := read(os.path.join(directory, "config.json"), json.loads)) or not isinstance(config := found[1], dict): + return False + return abs(found[0] - run["created_at"]) <= MATCH_SECONDS and config.get("model_name") == run["model"] + + +def checkpoint_runs(directory: str) -> set[str]: + found = read(os.path.join(directory, "checkpoints.jsonl"), lambda text: set(TINKER_PATH.findall(text))) + return found[1] if found else set() + + +def match(run: dict, candidates: list[str]) -> str | None: + """The run's directory: the one whose checkpoints.jsonl names tinker:///, else the one started() for it. Of + several, only one that was also started() for it is taken; otherwise none, never a guess.""" + exact = [d for d in candidates if run["run_id"] in checkpoint_runs(d)] + if len(exact) == 1: + return exact[0] + timed = [d for d in exact or candidates if started(d, run)] + return timed[0] if len(timed) == 1 else None + + +def metrics(directory: str) -> dict[int, dict[str, float]]: + """metrics.jsonl's finite numbers by each line's "step", reading only what was appended since the last call.""" + path = os.path.join(directory, "metrics.jsonl") + try: + stat = os.stat(path) + except FileNotFoundError: + return {} # no step has finished yet + inode, done, rows = progress.get(path, (stat.st_ino, 0, {})) + if inode != stat.st_ino or stat.st_size < done: # replaced or truncated + done, rows = 0, {} + with open(path, "rb") as f: + f.seek(done) + data = f.read(stat.st_size - done) + data = data[: data.rfind(b"\n") + 1] # a line still being written waits for the next read + rows = dict(rows) + for row in map(json.loads, data.splitlines()): + if isinstance(step := row.pop("step", None), int): # a call to log_metrics without a step has none + rows[step] = rows.get(step, {}) | finite(row) + progress[path] = (stat.st_ino, done + len(data), rows) + return rows + + +def log_page(directory: str, before: int | None, limit: int, q: str = "", severity: str | None = None) -> dict: + """logs.log, newest first, in the shape of a Cloud Logging page. Its lines carry no time, so a record is a header line + and the lines under it, such as a traceback, and next_cursor is the byte offset the next older page ends at.""" + path = os.path.join(directory, "logs.log") + with open(path, "rb") as f: + size = f.seek(0, os.SEEK_END) + end = size if before is None else min(before, size) + start = max(0, end - SCAN_BYTES) + f.seek(start) + block = f.read(end - start) + if start and (cut := block.find(b"\n") + 1): # the first line began before the block + block, start = block[cut:], start + cut + lines, at = [], start + for raw in block.splitlines(keepends=True): + lines.append((at, raw.decode("utf-8", "replace").rstrip("\r\n"))) + at += len(raw) + records, under, cursor = [], [], None + for at, text in reversed(lines): + under.append(text) + # The file's first line starts a record, and so does a block's first line when the block has no header at all. + if not (header := HEADER.match(text)) and at and not (at == start and cursor is None): + continue + message, under, cursor = "\n".join(reversed(under)), [], at + level = header[1] if header else "UNKNOWN" + if (not severity or level == severity) and q.lower() in message.lower(): + record = {"id": f"logs.log:{at}", "timestamp": None, "pod": "client", "role": "client", "node": None, "severity": level} + records.append(record | {"message": message[:MESSAGE_CHARS], "message_truncated": len(message) > MESSAGE_CHARS}) + if len(records) == limit: + break + return {"records": records, "next_cursor": str(cursor) if cursor else None} diff --git a/src/server/dashboard/facts.py b/src/server/dashboard/facts.py new file mode 100644 index 000000000..582d370af --- /dev/null +++ b/src/server/dashboard/facts.py @@ -0,0 +1,295 @@ +"""Facts about a run, joined from its records, with no verdicts. A block appears only when it has content, so an absent +block means nothing of its kind was recorded. A log pointer is a read against /api/v1/inspect to run as given.""" + +import asyncio +import statistics +from datetime import UTC, datetime + +from server.dashboard import fleet +from server.store import get_state_store +from server.telemetry import alive, key + +BUCKETS = {"train": {"forward_backward", "optim_step"}, "checkpoint": {"save_state", "save_weights_for_sampler"}, "forward": {"forward"}} + + +async def context(state: dict, run: dict, low: float = 0, high: float = float("inf")) -> dict: + """What the facts read about one run: its ops in [low, high], its workers, request and sample counters and queue.""" + store, run_id, finished = get_state_store(), run["run_id"], run["lifecycle"] == "ended" + ops, counts, queued, samples_queued, sampled, sampled_at = await asyncio.gather( + fleet.operations(run_id), + store.read_counts(key("ledger", run_id)), + store.list_length(request_queue(run_id)), + queued_samples(store, run, finished), + store.get_value(key("sampled", run_id)), + store.get_value(key("sampled_at", run_id)), + ) + ops = [s for s in ops if low <= s["at"] <= high] + trainer = [s for s in ops if s["operation"] != "sample"] # a single-process trainer answers samples too + ctx = { + "run": run, + "ops": ops, + "trainer": trainer, + "steps": [s for s in trainer if s["operation"] == "optim_step" and s.get("status") == "succeeded"], + } + ctx.update(ledger=counts, queued=queued, samples_queued=samples_queued, now=state["observed_at"], finished=finished, window=(low, high)) + ctx.update(sampled=int(sampled) if sampled else None, sampled_at=float(sampled_at) if sampled_at else None) + ctx.update(lanes=fleet.lanes(state, run_id), workers=workers_of(state, run)) + return ctx + + +def request_queue(run_id: str) -> str: + return f"open_rl:queue:{run_id}" # RedisStore's names for the queues it pushes to + + +def sampler_queue(runtime_id: str) -> str: + return f"open_rl:sampler_queue:{runtime_id}" + + +def workers_of(state: dict, run: dict) -> list[dict]: + return [w for w in state["workers"] if w.get("runtime_id") in (run["runtime_id"], run["run_id"])] + + +async def queued_samples(store, run: dict, finished: bool) -> int | None: + # Samples wait in the runtime's queue, which co-tenant runs share; a finished run's share of it is gone. + return None if finished else await store.list_length(sampler_queue(run["runtime_id"])) + + +def read(route: str, **args) -> dict: + return {"read": route, "args": {k: v for k, v in args.items() if v is not None}} + + +def iso(at: float) -> str: + return datetime.fromtimestamp(at, UTC).isoformat(timespec="seconds") + + +def present(**blocks) -> dict: + return {k: v for k, v in blocks.items() if v not in (None, [], {})} + + +def unanswered(ctx: dict) -> list[dict]: + """Trainer requests the client waits on, with the pod holding each and whether its worker still reports.""" + found = [] + for w in ctx["workers"]: + for g in (g for g in w.get("outstanding") or [] if g.get("run_id") == ctx["run"]["run_id"] and g["op"] != "sample"): + found.append({"op": g["op"], "count": g["count"], "age_s": round(ctx["now"] - g["since"]), "pod": w.get("pod") or w["id"]}) + found[-1]["holder_alive"] = alive(w, ctx["now"]) + return found + + +def failures(ctx: dict) -> dict | None: + if not (failed := [s for s in ctx["trainer"] if s.get("status") == "failed"]): + return None + ops = [{"operation": s["operation"], "error_type": s.get("error_type"), "at": iso(s["at"]), "pod": s.get("pod")} for s in reversed(failed[-5:])] + last, run_id = failed[-1], ctx["run"]["run_id"] + logs = read("logs", run_id=run_id, pod=last.get("pod"), q=last.get("error_type"), since=iso(last["started_at"] - 60), until=iso(last["at"] + 60)) + return {"count": len(failed), "ops": ops, "logs": logs} + + +def relaunches(ctx: dict) -> list[dict]: + """Worker processes that replaced an earlier one in the same pod during this run: a container restarted in place, or a + new pod after one died holding requests. A clean exit, or a shared runtime's idle pod recreated, is not a crash.""" + by_pod: dict[str, list[dict]] = {} + low, high = (ctx["run"]["created_at"] or 0) - 60, ctx["run"].get("completed_at") or float("inf") + for w in sorted(ctx["workers"], key=lambda w: w["started_at"]): + if w.get("pod") and low <= w["started_at"] <= high: + by_pod.setdefault(w["pod"], []).append(w) + found = [ + {"pod": b["pod"], "stopped_at": a.get("heartbeat_at", b["started_at"]), "started_at": b["started_at"]} + for p in by_pod.values() + for a, b in zip(p, p[1:]) + if a.get("phase") != "exited" and (a.get("pod_uid") == b.get("pod_uid") or any(g["op"] != "sample" for g in a.get("outstanding") or [])) + ] + return sorted(found, key=lambda p: p["started_at"]) + + +def saves(ctx: dict) -> dict | None: + if not (found := [s for s in ctx["trainer"] if s["operation"] == "save_state"]) and not ctx["steps"]: + return None + last = max((s["at"] for s in found if s.get("status") == "succeeded"), default=None) + return present( + count=len(found), + failed=sum(s.get("status") == "failed" for s in found), + last_at=iso(last) if last else None, + steps_since_last=sum(s["at"] > (last or 0) for s in ctx["steps"]), + slowest_s=round(max(s.get("elapsed_seconds", 0) for s in found)) if found else None, + ) + + +def sends(ctx: dict) -> int: + """The samples the run sent. A run recorded before the sample count kept a record per send.""" + return ctx["sampled"] if ctx["sampled"] is not None else sum(s["operation"] == "sample" for s in ctx["ops"]) + + +def answered(ctx: dict) -> list[dict]: + """The answered samples that kept a record: a torch trainer's, or a group a vLLM sampler served, timed as a whole.""" + return [s for s in ctx["ops"] if s["operation"] == "sample" and s.get("status") == "succeeded"] + + +def samplers(ctx: dict) -> dict | None: + """Sends and failures from the run's own count and records, and the median time of the groups its samplers served.""" + if not (sent := sends(ctx)): + return None + records = [s for s in ctx["ops"] if s["operation"] == "sample"] + found = { + "sends": sent, + "cancelled": sum(s.get("status") == "cancelled" for s in records), + "failed": sum(s.get("status") == "failed" for s in records), + } + if recorded := answered(ctx): + found["median_s"] = round(statistics.median(s.get("elapsed_seconds", 0) for s in recorded), 2) + return found + + +def config(ctx: dict) -> list[dict] | str | None: + """Workers whose echoed settings differ from what the run asked for.""" + if not (echoed := [w for w in ctx["workers"] if w.get("config")]): + return "no echo" + settings, found = ctx["run"]["settings"], [] + for w in echoed: + asked, running = settings.get("openrl.fine_tuning_type"), w["config"].get("env", {}).get("OPEN_RL_FINE_TUNING_TYPE") + if asked and running and asked != running: + found.append({"pod": w.get("pod"), "setting": "openrl.fine_tuning_type", "asked": asked, "running": running}) + # gpus is None when the worker could not count them. + gpus, wanted = w.get("gpus"), settings.get("openrl.trainer_gpus") + if w.get("role") == "trainer" and wanted and gpus is not None and len(gpus) != int(wanted): + found.append({"pod": w.get("pod"), "setting": "openrl.trainer_gpus", "asked": wanted, "running": len(gpus)}) + return found or None + + +def trainer_span(ctx: dict) -> tuple[float, float] | None: + """From the run's first trainer op to its last once the run has ended, else to now.""" + if not (ops := ctx["trainer"]): + return None + start = ops[0]["started_at"] + return start, start + max((ops[-1]["at"] if ctx["finished"] else ctx["now"]) - start, 1.0) + + +def goodput(ctx: dict) -> dict | None: + """Share of the run's span the trainer spent training, and where the rest went (Google ML goodput).""" + if not (span := trainer_span(ctx)): + return None + ops, (start, end) = ctx["trainer"], span + wall = end - start + spent = {kind: sum(s.get("elapsed_seconds", 0) for s in ops if s["operation"] in names) for kind, names in BUCKETS.items()} + idle = max(0.0, wall - sum(spent.values())) + split = {"other": round(idle)} + # Trainer idle time with a sample in flight is rollout; with none the server waits on the client. + if running := [(s["started_at"], s["at"]) for s in answered(ctx)]: + busy = [(s["started_at"], s["at"]) for s in ops] + rollout = min(idle, max(0.0, covered(busy + running, start, end) - covered(busy, start, end))) + split = {"rollout": round(rollout), "client": round(idle - rollout)} + first = round(ctx["steps"][0]["at"] - start) if ctx["steps"] else None + return { + "goodput": round(spent["train"] / wall, 3), + "seconds": {"wall": round(wall), **{k: round(v) for k, v in spent.items()}, **split}, + "first_step_after_s": first, + } + + +def covered(spans: list, low: float, high: float) -> float: + """Seconds of [low, high] inside at least one of the spans.""" + return sum(b - a for a, b in fleet.merge([(max(a, low), min(b, high)) for a, b in spans if b > low and a < high])) + + +def last_op_at(ctx: dict) -> float | None: + """When the run last did anything in the context's window: its last recorded op, or its last sample.""" + low, high = ctx["window"] + sampled = [at for at in [ctx["sampled_at"]] if at is not None and low <= at <= high] + return max([s["at"] for s in ctx["ops"][-1:]] + sampled, default=None) + + +def summary(ctx: dict) -> dict: + steps, ops, run, ledger = ctx["steps"], ctx["trainer"], ctx["run"], dict(ctx["ledger"]) + gaps = [b["at"] - a["at"] for a, b in zip(steps, steps[1:])] + if "enqueued" in ledger and ctx["queued"] is not None: + ledger["lost"] = ledger["enqueued"] - ledger.get("popped", 0) - ctx["queued"] + poisoned = [s for s in ops if s.get("nonfinite")] + first = present(operation=poisoned[0]["operation"], step=poisoned[0].get("step"), metrics=poisoned[0]["nonfinite"]) if poisoned else None + norms = [float(f"{s['metrics']['grad_norm:mean']:.4g}") for s in steps if "grad_norm:mean" in s.get("metrics", {})] + restarts = None + if relaunched := relaunches(ctx): + p = relaunched[0] + logs = read("logs", run_id=run["run_id"], pod=p["pod"], since=iso(p["stopped_at"] - 120), until=iso(p["started_at"] + 60)) + restarts = { + "workers": [{"pod": p["pod"], "stopped_reporting_at": iso(p["stopped_at"]), "restarted_at": iso(p["started_at"])} for p in relaunched], + "logs": logs, + # Who created or deleted the run's Workloads and pods around the relaunches, from the cluster's audit log. + "cluster_changes": ctx.get("changes"), + } + unplaced = [ + {"name": w["name"], "role": w["role"], "age_s": round(w["age_seconds"] or 0), "message": w["message"]} for w in run["workloads"] if not w["node"] + ] + # A live pod's restarts, and the terminations placement records kept after their pods were gone. + kept = [ + {"at": t["at"], "pod": lane["pod"], "reason": t.get("reason") or "Terminated", "message": exit_text(t)} + for lane in ctx["lanes"] + for t in lane.get("terminations") or [] + ] + found = {(i["pod"], round(i["at"])): i for i in [*kept, *(i for pod in run["pods"] for i in pod["incidents"])]} + incidents = sorted(found.values(), key=lambda i: -i["at"])[:5] + return present( + lifecycle=run["lifecycle"], + steps=len(steps), + step_seconds={"median": round(statistics.median(gaps), 1), "last": round(gaps[-1], 1)} if gaps else None, + since_last_step_s=round(ctx["now"] - steps[-1]["at"]) if steps and not ctx["finished"] else None, + since_last_op_s=max(0, round(ctx["now"] - last)) if (last := last_op_at(ctx)) is not None and not ctx["finished"] else None, + trainer_idle_s=max(0, round(ctx["now"] - ops[-1]["at"])) if ops and not ctx["finished"] else None, + last_op={"operation": ops[-1]["operation"], "status": ops[-1].get("status"), "at": iso(ops[-1]["at"])} if ops else None, + last_metrics={k: float(f"{v:.6g}") for s in ops[-50:] for k, v in s.get("metrics", {}).items()}, + goodput=goodput(ctx), + requests=present(unanswered=unanswered(ctx), ledger=ledger, samples_queued=ctx["samples_queued"] or None), + failures=failures(ctx), + relaunches=restarts, + saves=saves(ctx), + optimizer=present(nonfinite={"count": len(poisoned), "first": first} if poisoned else None, grad_norm_last=norms[-5:]), + samplers=samplers(ctx), + config=config(ctx), + placement=present(recorded=len(ctx["lanes"]), unplaced=unplaced) or None, + incidents=[present(at=iso(i["at"]), pod=i["pod"], reason=i["reason"], message=(i.get("message") or "")[:200]) for i in incidents], + ) + + +def exit_text(termination: dict) -> str: + signal = f", signal {termination['signal']}" if termination.get("signal") else "" + return f"{termination.get('container') or 'container'} exited with code {termination.get('exit_code')}{signal}" + + +def line(ctx: dict) -> dict: + """A run in one line from its full context, for a session over the session's span.""" + failures = sum(s.get("status") == "failed" for s in ctx["trainer"]) + return one_line(ctx, len(ctx["steps"]), [s["at"] for s in ctx["steps"]], last_op_at(ctx), failures) + + +async def fleet_line(state: dict, run: dict) -> dict: + """A run in one line for /runs from a few small records, never its command history. A run recorded before step times + and the failure count existed leaves those null.""" + store, run_id, finished = get_state_store(), run["run_id"], run["lifecycle"] == "ended" + counts, last_ops, last_samples, times, samples_queued, sampled_at = await asyncio.gather( + store.read_counts(key("ledger", run_id)), + store.read_samples(key("operations", run_id), last=1), + store.read_samples(key("operations", run_id, "samples"), last=1), + store.read_samples(key("step_times", run_id)), + queued_samples(store, run, finished), + store.get_value(key("sampled_at", run_id)), + ) + ctx = {"run": run, "workers": workers_of(state, run), "now": state["observed_at"], "samples_queued": samples_queued} + last = max([r["at"] for r in last_ops + last_samples] + ([float(sampled_at)] if sampled_at else []), default=None) + return one_line(ctx, run["steps"], times, last, counts.get("failed", 0) if counts else None) + + +def one_line(ctx: dict, steps: int, step_times: list[float], last_op_at: float | None, failures: int | None) -> dict: + """A run in one line, enough to see which run to open. ctx holds the run, its workers, now and its queued samples.""" + held, now = unanswered(ctx), ctx["now"] + return { + "lifecycle": ctx["run"]["lifecycle"], + "steps": steps, + # The snapshot time lags records written after it, so a fresh op can look a few seconds in the future. + "since_last_op_s": max(0, round(now - last_op_at)) if last_op_at is not None else None, + "since_last_step_s": round(now - step_times[-1]) if step_times else None, + "median_step_s": round(statistics.median(b - a for a, b in zip(step_times, step_times[1:]))) if len(step_times) > 1 else None, + "trainer_requests_unanswered": sum(o["count"] for o in held), + "held_by_dead_worker": sum(o["count"] for o in held if not o["holder_alive"]), + "samples_queued": ctx["samples_queued"], + "failures": failures, + "relaunches": len(relaunches(ctx)), + } diff --git a/src/server/dashboard/fleet.py b/src/server/dashboard/fleet.py new file mode 100644 index 000000000..25a5d1738 --- /dev/null +++ b/src/server/dashboard/fleet.py @@ -0,0 +1,196 @@ +"""The one view every page and fact reads: runs joined to their pods and workloads, worker and placement records, and nodes.""" + +import asyncio +import json +import time +from collections import OrderedDict + +from server import openrl_status +from server.dashboard import sources +from server.model_metadata import decode_model_metadata +from server.store import get_state_store +from server.telemetry import key +from server.worker_manager import STOPPED, WAITING_FOR_PLACEMENT, owner_id + +CACHE_SECONDS, OPS_SECONDS, CACHE_ENTRIES = 5, 10, 32 +cache: OrderedDict[str, dict] = OrderedDict() # least recently used first +STALE_SECONDS = 60 + + +def succeeded(task: asyncio.Future | None) -> bool: + return task is not None and task.done() and not task.cancelled() and task.exception() is None + + +async def cached(name: str, seconds: float, read) -> object: + """One read serves every caller. Once it is older than `seconds`, or failed, a fresh read starts and callers get the last + good result until it lands, unless that is far older than the window. Only the CACHE_ENTRIES most recently used are kept.""" + entry = cache.setdefault(name, {"at": 0.0, "task": None, "good": None, "good_at": 0.0}) + cache.move_to_end(name) + while len(cache) > CACHE_ENTRIES: + cache.popitem(last=False) + task = entry["task"] + if succeeded(task): + entry["good"], entry["good_at"] = task, entry["at"] + if task is None or (task.done() and (not succeeded(task) or time.monotonic() - entry["at"] > seconds)): + entry["at"], entry["task"] = time.monotonic(), asyncio.ensure_future(read()) + entry["task"].add_done_callback(lambda t: t.cancelled() or t.exception()) # retrieved, so asyncio does not log it + if succeeded(entry["task"]): + return entry["task"].result() + # A result much older than its window is no answer: after an idle spell the caller waits for the fresh read. + if entry["good"] is not None and time.monotonic() - entry["good_at"] <= max(STALE_SECONDS, 6 * seconds): + return entry["good"].result() + return await asyncio.shield(entry["task"]) + + +async def snapshot() -> dict: + return await cached("fleet", CACHE_SECONDS, read) + + +async def operations(run_id: str) -> list[dict]: + """A run's command records, trainer ops and failed samples, oldest first.""" + + async def read(): + store = get_state_store() + ops, samples = await asyncio.gather(store.read_samples(key("operations", run_id)), store.read_samples(key("operations", run_id, "samples"))) + return sorted(ops + samples, key=lambda r: r["at"]) + + return await cached(run_id, OPS_SECONDS, read) + + +async def node_gpus() -> dict[str, list[dict]]: + """Each node's GPUs now, from DCGM: its UUID and slot, in slot order.""" + found: dict[str, list[dict]] = {} + now = time.time() + for row in await sources.prom(f'max by (Hostname, UUID, gpu) (DCGM_FI_DEV_GPU_UTIL{{UUID!=""{sources.scope()}}})', now, now): + found.setdefault(row["metric"].get("Hostname"), []).append({"id": row["metric"]["UUID"], "name": f"gpu-{row['metric'].get('gpu')}"}) + return {node: sorted(gpus, key=lambda d: (len(d["name"]), d["name"])) for node, gpus in found.items()} + + +def lifecycle(run: dict, live: bool, workloads: list | None) -> str: + """ended: finished, or nothing serves it and no session is live. waiting: a worker is not placed yet. + placed: every worker is placed. active: the scheduler cannot be read, so placement is unknown.""" + states = {w["state"] for w in run["workloads"]} + if run["status"] != "active" or (states and states <= {STOPPED}) or not (states or live): + return "ended" + return "active" if workloads is None else "waiting" if not states or WAITING_FOR_PLACEMENT in states else "placed" + + +def join(run_id: str, raw: str, steps: dict, live: set, cluster: dict) -> dict: + """A run is its model. LoRA runs may share a base-model runtime; an FFT or exclusive run's runtime is the run itself.""" + row, meta = json.loads(raw), decode_model_metadata(raw) + runtime, kind = meta.runtime(run_id), "lora" if meta.fine_tuning_type == "lora" else "fft" + shared = runtime != run_id + # A finished run on a shared runtime lets go of it; the runtime's pods and workloads now serve others. + keep = meta.status == "active" or not shared + owner = owner_id(runtime).rstrip("-") + pods = [p for p in cluster["pods"] if keep and p["owner"] == owner] + workloads = [w for w in cluster["workloads"] or [] if keep and w["model_id"] == runtime and w["training_kind"] == kind] + run = {"run_id": run_id, "name": row.get("name") or row.get("run_name") or f"{(meta.base_model or 'Run').split('/')[-1]} · {run_id[:8]}"} + run.update(model=meta.base_model, display_name=row.get("name") or row.get("run_name"), recipe_name=row.get("recipe_name")) + run.update(fine_tuning_type=meta.fine_tuning_type, status=meta.status, created_at=meta.created_at, completed_at=meta.completed_at) + run.update(steps=int(steps.get(key("steps", run_id), 0)), settings=openrl_status.settings_of(meta), session_ids=meta.session_ids) + run.update(runtime_id=runtime, shared_runtime=shared, pods=pods, workloads=workloads, nodes=sorted({p["node"] for p in pods if p["node"]})) + run["lifecycle"] = lifecycle(run, any(s in live for s in meta.session_ids), cluster["workloads"]) + return run + + +async def read() -> dict: + store, now = get_state_store(), time.time() + cluster, gpus = await asyncio.gather(sources.kubernetes(), node_gpus(), return_exceptions=True) + if isinstance(gpus, Exception): + gpus, cluster["errors"] = {}, [*cluster["errors"], f"GPU inventory unavailable: {gpus}"] + metas, steps, workers, live = await asyncio.gather(*(store.scan_values(key(kind, "")) for kind in ("model_meta", "steps", "worker", "session"))) + live = {name.removeprefix(key("session", "")) for name in live} + runs = sorted( + (join(name.removeprefix(key("model_meta", "")), raw, steps, live, cluster) for name, raw in metas.items()), key=lambda r: -r["created_at"] + ) + placements = await asyncio.gather(*(store.read_samples(key("placements", runtime)) for runtime in {r["runtime_id"] for r in runs})) + + async def last_op(run): # when a run that never said it finished last did anything + run["last_completed_at"] = max((s["at"] for s in await store.read_samples(key("operations", run["run_id"]), last=1)), default=None) + + await asyncio.gather(*(last_op(run) for run in runs if run["status"] == "active")) + nodes = [{**n, "devices": gpus.get(n["name"], []), "gpu_capacity": max(n["gpu_capacity"], len(gpus.get(n["name"], [])))} for n in cluster["nodes"]] + fleet = {"observed_at": now, "runs": runs, "workers": [json.loads(w) for w in workers.values()], "placements": [p for ps in placements for p in ps]} + return fleet | {"nodes": nodes, "pods": cluster["pods"], "workloads": cluster["workloads"], "errors": cluster["errors"]} + + +def lanes(fleet: dict, run_id: str | None = None, node_id: str | None = None) -> list[dict]: + """One lane per placement record, with its worker process's phase and GPU turns. A lane on a shared runtime + becomes one lane per run, which shows only that run's ops.""" + found = [] + for record in fleet["placements"]: + start, end = record["placed_at"], record["released_at"] or fleet["observed_at"] + members = [r for r in fleet["runs"] if r["runtime_id"] == record["model_id"] and r["created_at"] <= end and (r["completed_at"] or end) >= start] + # Pod names repeat across placements of one Workload; the process ran inside this placement's span. + procs = [p for p in fleet["workers"] if record["pod"] and p.get("pod") == record["pod"] and p["started_at"] <= end and p["heartbeat_at"] >= start] + ids = [r["run_id"] for r in members] + lane = {"id": f"{record['workload']}@{record['placed_at']:.0f}", "name": record["workload"], "run_ids": ids, "runtime_id": record["model_id"]} + lane.update(label=members[0]["name"] if len(members) == 1 else record["model_id"], shared_runtime=record["model_id"] not in ids if ids else None) + lane.update(role=record["role"], node=record["node"] or "", pod=record["pod"], devices=record["gpu_uuids"], device_count=len(record["gpu_uuids"])) + lane.update(phase=max(procs, key=lambda p: p["started_at"])["phase"] if procs else None, turns=[t for p in procs for t in p.get("turns") or []]) + lane.update(waits=[w for p in procs for w in p.get("waits") or []], terminations=record.get("terminations") or []) + lane.update(start=start, end=end, ended=record["released_at"] is not None) + for member in members if lane["shared_runtime"] else [None]: + if member is None: + found.append(lane) + else: + label = f"{(member['model'] or lane['label']).split('/')[-1]} · {member['run_id'][:8]}" + found.append({**lane, "id": f"{lane['id']}:{member['run_id']}", "allocation_id": lane["id"], "run_ids": [member["run_id"]], "label": label}) + return [p for p in found if (not run_id or run_id in p["run_ids"]) and (not node_id or p["node"] == node_id)] + + +def merge(spans: list) -> list: + result = [] + for start, end in sorted(spans): + if result and start <= result[-1][1]: + result[-1][1] = max(end, result[-1][1]) + else: + result.append([start, end]) + return result + + +async def activity(fleet: dict, low: float, high: float, run_id: str | None = None, node_id: str | None = None) -> list[dict]: + """Each lane in the window with its GPU turns and waits for them (none on a shared lane) and the ops it ran, clipped to it. A sampler lane's + samples are the groups it served and its failed sends. Never utilization.""" + shown = [p for p in lanes(fleet, run_id, node_id) if p["start"] <= high and p["end"] >= low] + wanted = sorted({n for lane in shown for n in lane["run_ids"]}) + records = dict(zip(wanted, await asyncio.gather(*(operations(n) for n in wanted)))) + for lane in shown: + lo, hi, turns, waits = max(low, lane["start"]), min(high, lane["end"]), lane.pop("turns"), lane.pop("waits") + turns = [] if lane["shared_runtime"] else [[max(lo, a), min(hi, b)] for a, b in turns if a < hi and b > lo] + # A grant inside half a second is the slicer's own round trip, not a wait for another holder. + waits = [] if lane["shared_runtime"] else [[max(lo, a), min(hi, b)] for a, b in waits if a < hi and b > lo and b - a >= 0.5] + ops: dict[str, list] = {} + for r in (r for n in lane["run_ids"] for r in records[n]): + start = max(lo, r.get("started_at") or r["at"] - r.get("elapsed_seconds", 0)) + # A sample lands on the replica pod that answered it. + if r.get("role") == lane["role"] and r.get("pod") in (None, lane["pod"]) and start < min(hi, r["at"]): + ops.setdefault(r["operation"], []).append([start, min(hi, r["at"])]) + lane.update(ops={n: merge(s) for n, s in ops.items()}, exact=bool(turns), intervals=merge(turns + [s for group in ops.values() for s in group])) + lane["waits"] = merge(waits) + return shown + + +def steps(ops: list[dict]) -> list[dict]: + """Trainer ops grouped by the step they belong to: the step's window, and per op its count, time and metric spread.""" + found: dict[int, dict] = {} + completed = 0 # records from before ops carried their step count optim_steps instead + for s in (s for s in ops if s["operation"] != "sample"): + number = s["step"] if isinstance(s.get("step"), int) else completed + completed += s["operation"] == "optim_step" and s.get("status") == "succeeded" + step = found.setdefault(number, {"step": number, "started_at": s["started_at"], "ended_at": s["at"], "ops": {}, "nonfinite": set()}) + step["started_at"], step["ended_at"] = min(step["started_at"], s["started_at"]), max(step["ended_at"], s["at"]) + op = step["ops"].setdefault(s["operation"], {"count": 0, "failed": 0, "seconds": 0.0, "metrics": {}}) + op["count"] += 1 + op["failed"] += s.get("status") != "succeeded" + op["seconds"] += s.get("elapsed_seconds", 0) + for name, value in s.get("metrics", {}).items(): + op["metrics"].setdefault(name, []).append(value) + step["nonfinite"].update(s.get("nonfinite", ())) + for step in found.values(): + optim = step["ops"].get("optim_step", {"count": 0, "failed": 0}) + step.update(nonfinite=sorted(step["nonfinite"]), complete=optim["count"] > optim["failed"]) + for op in step["ops"].values(): + op["metrics"] = {name: {"min": min(v), "max": max(v), "mean": sum(v) / op["count"]} for name, v in op["metrics"].items()} + return [found[k] for k in sorted(found)] diff --git a/src/server/dashboard/router.py b/src/server/dashboard/router.py new file mode 100644 index 000000000..753010d70 --- /dev/null +++ b/src/server/dashboard/router.py @@ -0,0 +1,370 @@ +"""The operator UI and the same read-only JSON API for agents, under /api/v1/inspect.""" + +import asyncio +import json +import math +from pathlib import Path +from string import Template +from typing import Annotated + +import httpx +from fastapi import APIRouter, HTTPException, Query +from fastapi.responses import FileResponse +from pydantic import BaseModel + +from server.dashboard import client_runs, facts, fleet, sources +from server.store import get_state_store +from server.telemetry import finite, key + +router = APIRouter(tags=["inspection"]) +STATIC = Path(__file__).parent / "static" +BOUNDS = ( + "ISO-8601, e.g. 2026-01-02T12:00:00Z, or Unix seconds; since also takes a span before until, e.g. 1h. " + "Default: the 30 minutes ending now; at most 7 days." +) +Bound = Annotated[str | None, Query(max_length=64, description=BOUNDS)] + + +class Doc(BaseModel, extra="allow", allow_inf_nan=False): + """Declared fields are always present; the rest pass through as recorded. Log messages are untrusted output.""" + + +@router.get("/dashboard", include_in_schema=False) +@router.get("/dashboard/", include_in_schema=False) +@router.get("/dashboard/assets/{name}", include_in_schema=False) +async def dashboard(name: str = "index.html"): + if (path := (STATIC / name).resolve()).parent != STATIC.resolve() or not path.is_file(): + raise HTTPException(404, "Not found") + return FileResponse(path, headers={"Cache-Control": "no-store"}) + + +def span(since: str | None, until: str | None) -> tuple[str, str, float, float]: + try: + return sources.window(since, until) + except (ValueError, OverflowError, OSError) as exc: + raise HTTPException(400, "Use ISO-8601 or Unix-second since/until and a positive window of at most seven days") from exc + + +async def run_of(run_id: str) -> tuple[dict, dict]: + """The run, or the one run of a session named in its place.""" + state = await fleet.snapshot() + found = [r for r in state["runs"] if r["run_id"] == run_id] or [r for r in state["runs"] if run_id in r["session_ids"]] + if len(found) > 1: + raise HTTPException(409, f"Session {run_id} has {len(found)} runs; name one of {', '.join(sorted(r['run_id'] for r in found))}") + if not found: + raise HTTPException(404, f"No run {run_id}") + return state, found[0] + + +@router.get("/api/v1/inspect/inventory", response_model=Doc) +async def inventory(): + """Runs with their lifecycle (ended, waiting, placed, active), nodes with their GPUs, the GPUs held now, pods with + problems, and an error per unavailable source. A run's facts are /runs/{id}/summary.""" + await sources.discover() + state = await fleet.snapshot() + hidden = ("workloads", "runtime_id", "pods") + # status is what the client last reported (active until it deletes the model), so a crashed run stays active there; + # lifecycle is what the cluster shows. + runs = [ + {("client_status" if k == "status" else k): v for k, v in r.items() if k not in hidden} + | {"pods": [{"name": p["name"], "problem": p["problem"]} for p in r["pods"]]} + for r in state["runs"] + ] + fields = ("id", "label", "run_ids", "role", "node", "pod", "devices", "device_count") + held = [{k: lane[k] for k in fields} for lane in fleet.lanes(state) if not lane["ended"]] + pods = [{"name": p["name"], "problem": p["problem"], "restarts": p["restarts"]} for p in state["pods"] if p["problem"]] + cluster = {"available": not any(e.startswith(("Kubernetes", "pods")) for e in state["errors"]), "nodes": state["nodes"], "pods": pods} + return {"observed_at": facts.iso(state["observed_at"]), "errors": state["errors"], "runs": runs, "cluster": cluster, "placements": held} + + +@router.get("/api/v1/inspect/activity", response_model=Doc) +async def activity(since: Bound = None, until: Bound = None, run_id: str | None = None, node_id: str | None = None): + """Each placement's GPU turns, waits (from asking the time slicer for the GPU to the grant, half a second or more), recorded + ops and its pod's container terminations in the window (Unix seconds), fleet-wide, a node's or a run's. + A sampler's `sample` spans are the groups of samples it served, one record per run each. Gaps do not prove idleness.""" + since, until, low, high = span(since, until) + if run_id: + state, run = await run_of(run_id) + run_id = run["run_id"] + else: + state = await fleet.snapshot() + placements = await fleet.activity(state, low, high, run_id, node_id) + return {"placements": placements, "since": since, "until": until, "observed_at": facts.iso(state["observed_at"])} + + +VLLM = "{pod=~$pods$scope}" + + +# The DCGM exporter's collection interval in k8s/deploy. +DCGM_SECONDS = 10 + + +def utilization(form: str, by: str = "UUID, gpu") -> str: + """A GPU's busy percent. GR_ENGINE_ACTIVE averages DCGM's whole collection interval. GPU_UTIL, the fallback where it is + missing, covers about the last second of each interval, so a load with that period aliases.""" + active, util = "DCGM_FI_PROF_GR_ENGINE_ACTIVE{UUID=~$gpus$scope}", "DCGM_FI_DEV_GPU_UTIL{UUID=~$gpus$scope}" + return f"max by ({by}) (100 * {form.format(active)}) or max by ({by}) ({form.format(util)})" + + +QUERIES = { + # By UUID and gpu: DCGM's slot index names a GPU after its node is gone. + "gpu_util": utilization("{}"), + "gpu_memory_mib": "max by (UUID, gpu) (DCGM_FI_DEV_FB_USED{UUID=~$gpus$scope})", + "gpu_memory_peak_mib": "max by (UUID, gpu) (max_over_time(DCGM_FI_DEV_FB_USED{UUID=~$gpus$scope}[${window}s]))", + "gpu_memory_total_mib": "max by (UUID) (DCGM_FI_DEV_FB_TOTAL{UUID=~$gpus$scope})", + # Utilization comes in bursts; the last two minutes say whether a GPU is working now better than one sample. + "gpu_util_now": utilization("avg_over_time({}[2m])", "Hostname, UUID, gpu, modelName"), + # Duty cycle over the window: the share of time a kernel ran, one number per GPU with points=1. + "gpu_busy": utilization("avg_over_time({}[${{window}}s])", "Hostname, UUID, gpu, modelName"), + # From vLLM's and llm-d's published Grafana dashboards. + "vllm_generation_tokens_per_second": f"sum(rate(vllm:generation_tokens_total{VLLM}[2m]))", + "vllm_prompt_tokens_per_second": f"sum(rate(vllm:prompt_tokens_total{VLLM}[2m]))", + "vllm_requests_running": f"sum(vllm:num_requests_running{VLLM})", + "vllm_requests_waiting": f"sum(vllm:num_requests_waiting{VLLM})", + "vllm_kv_cache_usage": f"max(vllm:kv_cache_usage_perc{VLLM})", + "vllm_preemptions_per_second": f"sum(rate(vllm:num_preemptions_total{VLLM}[2m]))", + "vllm_time_to_first_token_p99_seconds": f"histogram_quantile(0.99, sum by (le) (rate(vllm:time_to_first_token_seconds_bucket{VLLM}[2m])))", + "vllm_request_latency_p99_seconds": f"histogram_quantile(0.99, sum by (le) (rate(vllm:e2e_request_latency_seconds_bucket{VLLM}[2m])))", +} + + +def subject(state: dict, run_id: str | None, node_id: str | None, low: float, high: float) -> tuple[list[str] | None, list[str]]: + """The GPUs and pods a run's or a node's placements held in the window, plus a node's GPUs now; None GPUs is the fleet.""" + if not (run_id or node_id): + return None, [] + held = [p for p in fleet.lanes(state, run_id, node_id) if p["start"] <= high and p["end"] >= low] + now = [d["id"] for n in state["nodes"] if not run_id and n["name"] == node_id for d in n["devices"]] + return sorted({uuid for p in held for uuid in p["devices"]} | set(now)), sorted({p["pod"] for p in held if p["pod"]}) + + +@router.get("/api/v1/inspect/query", response_model=Doc) +async def query( + name: Annotated[str | None, Query(description="One of the named queries; leave it out to list them")] = None, + run_id: str | None = None, + node_id: str | None = None, + since: Bound = None, + until: Bound = None, + points: int = Query(300, ge=1, le=1000), +): + """A named PromQL query over a run's GPUs and pods, a node's, or (GPU queries only) the fleet's, at `points` resolution, + `step` seconds apart. points=1 evaluates once over the whole window, which is how a duty cycle is one number per GPU. + A gpu_util point is a DCGM reading placed at the start of the reading_seconds it averaged, so it lines up with turns.""" + if name not in QUERIES: + raise HTTPException(400, f"{'Unknown query' if name else 'Name a query'}; use one of {sorted(QUERIES)}") + since, until, low, high = span(since, until) + await sources.discover() + gpus, pods = subject(await fleet.snapshot(), run_id, node_id, low, high) + step = sources.step_seconds(low, high, points) + result = {"name": name, "since": since, "until": until, "step": step, "series": [], "source": sources.prometheus_source()} + if gpus == [] and "$gpus" in QUERIES[name] or not pods and "$pods" in QUERIES[name]: + return {**result, "error": "No GPUs or pods recorded for this subject in this window"} + values = {"gpus": '".+"' if gpus is None else sources.any_of(gpus), "pods": sources.any_of(pods), "window": max(1, int(high - low))} + result["query"] = text = Template(QUERIES[name]).substitute(values, scope=sources.scope()) + try: + rows = await sources.prom(text, high if points == 1 else low, high, points) + except Exception as exc: + return {**result, "error": str(exc) if isinstance(exc, LookupError) else f"Query failed: {type(exc).__name__}"} + # A reading averages the interval before it was collected, and a range query returns it up to one reading later. + shift = 2 * DCGM_SECONDS if name == "gpu_util" else 0 + series = [{"labels": r["metric"], "points": [[float(t) - shift, float(v)] for t, v in r["values"] if math.isfinite(float(v))]} for r in rows] + return result | {"series": series} | ({"reading_seconds": DCGM_SECONDS} if shift else {}) + + +@router.get("/api/v1/inspect/accelerators", response_model=Doc) +async def accelerators(run_id: str | None = None, node_id: str | None = None, since: Bound = None, until: Bound = None): + """Every GPU in the fleet, a node's, or a run's in the window: node, slot, model, holders (the newest placement on it + and any time-sliced with it, with their pods' container terminations), busy (percent of the window DCGM saw the GPU + engine active), and peak and total memory in MiB. The window defaults to the last hour, or an ended run's last hour.""" + if run_id and since is None and until is None: + state, run = await run_of(run_id) + run_id, ends = run["run_id"], [p["end"] for p in fleet.lanes(state, run["run_id"])] + if run["lifecycle"] == "ended" and ends: + since, until = "1h", facts.iso(max(ends)) + since, until, low, high = span(since or ("1h" if until is None else None), until) + names = {"busy": "gpu_busy", "memory_peak_mib": "gpu_memory_peak_mib", "memory_total_mib": "gpu_memory_total_mib"} + found = dict(zip(names, await asyncio.gather(*(query(name, run_id, node_id, since, until, 1) for name in names.values())))) + result = {"since": since, "until": until, "gpus": [], "source": found["busy"]["source"]} + if error := next((r["error"] for r in found.values() if r.get("error")), None): + return {**result, "error": error} + value = {k: {s["labels"]["UUID"]: round(s["points"][-1][1], 1) for s in r["series"] if s["points"]} for k, r in found.items()} + held = sorted((p for p in fleet.lanes(await fleet.snapshot(), run_id) if p["start"] <= high and p["end"] >= low), key=lambda p: p["start"]) + placed: dict[str, list[dict]] = {} + for lane in held: + for uuid in lane["devices"]: + placed.setdefault(uuid, []).append(lane) + # A time-sliced GPU has several holders at once, every placement that overlaps the newest one. + holders = {uuid: [p for p in lanes if p["end"] >= lanes[-1]["start"]] for uuid, lanes in placed.items()} + labels_of = {series["labels"]["UUID"]: series["labels"] for series in found["busy"]["series"]} + for uuid, labels in labels_of.items(): + node = labels.get("Hostname") or next((lane["node"] for lane in reversed(holders.get(uuid, []))), None) + gpu = {"uuid": uuid, "node": node, "slot": labels.get("gpu"), "model": labels.get("modelName")} + gpu |= {name: value[name][uuid] for name in names if uuid in value[name]} + if uuid in holders: + gpu["holders"] = [{k: lane[k] for k in ("run_ids", "role", "pod", "start", "end", "ended", "terminations")} for lane in holders[uuid]] + result["gpus"].append(gpu) + result["gpus"].sort(key=lambda g: (g["node"] or "", int(g["slot"] or 0))) + return result + + +# Prefixes of the cookbook's metrics an agent reads first; the rest is a fields= away. +KEY_METRICS = ( + *("env/all/reward", "test/env/all/reward", "env/all/lab/criteria_pass_fraction", "test/env/all/lab/criteria_pass_fraction"), + *("optim/kl", "optim/entropy", "time/total", "progress/"), +) + + +async def client_directory(run: dict) -> str | None: + """The run's client directory under OPEN_RL_CLIENT_RUNS_ROOT, if exactly one matches it.""" + candidates = await fleet.cached("client directories", 30, lambda: asyncio.to_thread(client_runs.directories)) + return await asyncio.to_thread(client_runs.match, run, candidates) + + +@router.get("/api/v1/inspect/runs/{run_id}/steps", response_model=Doc) +async def steps( + run_id: str, + last: int = Query(20, ge=1, le=1000), + fields: Annotated[str | None, Query(description="Comma-separated prefixes of client metrics to keep, e.g. env/all/reward; * keeps all")] = None, +): + """The run's last steps, oldest first: window, completion, trainer ops (count, failures, seconds, metric spread), and the + `client` metrics its client wrote to metrics.jsonl for the step: by default the key ones (rewards, KL, entropy, step time, + progress), with every name the client logged in client_metric_names.""" + _, run = await run_of(run_id) + run_id = run["run_id"] + keep = () if fields == "*" else tuple(p for p in fields.split(",") if p) if fields else KEY_METRICS + every, result = fleet.steps(await fleet.operations(run_id)), {} + # Some runs keep their client metrics in Redis; metrics.jsonl wins a step both have. + client = {row["step"]: finite(row["metrics"]) for row in await get_state_store().read_samples(f"open_rl:run_metrics:{run_id}")} + if directory := await client_directory(run): + try: + client |= await asyncio.to_thread(client_runs.metrics, directory) + except (OSError, ValueError) as exc: # steps stand without the client's metrics + result["client_error"] = f"{directory}/metrics.jsonl unreadable: {type(exc).__name__}" + result["client_directory"] = directory + for step in every[-last:]: + step["client"] = {k: v for k, v in client.get(step["step"], {}).items() if not keep or k.startswith(keep)} + result["client_metric_names"] = sorted({name for metrics in client.values() for name in metrics}) + return result | {"run_id": run_id, "total": len(every), "steps": every[-last:]} + + +@router.get("/api/v1/inspect/runs/{run_id}/commands", response_model=Doc) +async def commands(run_id: str, since: Bound = None, until: Bound = None, role: str | None = None, limit: int = Query(100, ge=1, le=5000)): + """The run's raw command records (trainer ops, the groups of samples a sampler served, and failed samples), + newest first; without bounds, all of them. A group's record has its count of samples.""" + run_id = (await run_of(run_id))[1]["run_id"] + low, high = span(since, until)[2:] if since or until else (0, math.inf) + wanted = (lambda r: True) if role is None else (lambda r: (r.get("role") == "sampler") == (role == "sampler")) + found = [r for r in reversed(await fleet.operations(run_id)) if low <= r["at"] <= high and wanted(r)] + return {"run_id": run_id, "total": len(found), "commands": found[:limit]} + + +@router.get("/api/v1/inspect/runs", response_model=Doc) +async def runs(ended: bool = False): + """Every live run in one line, longest since its last op first, so a stuck run is one read away. An op includes a sample. + ended=true adds ended runs.""" + state = await fleet.snapshot() + chosen = [r for r in state["runs"] if ended or r["lifecycle"] != "ended"] + # A few runs at a time, since each read in flight holds a store connection. + limit = asyncio.Semaphore(128) + + async def line(run): + async with limit: + return await facts.fleet_line(state, run) + + found = await asyncio.gather(*(line(r) for r in chosen)) + lines = [{"run_id": r["run_id"], "name": r["name"], **line} for r, line in zip(chosen, found)] + return {"runs": sorted(lines, key=lambda r: -(r["since_last_op_s"] if r["since_last_op_s"] is not None else math.inf))} + + +@router.get("/api/v1/inspect/runs/{run_id}/summary", response_model=Doc) +async def summary(run_id: str): + """The run's facts, each block only when it has content: pace, goodput, unanswered requests and their holders, the ledger + (lost = enqueued - popped - queued), failures and relaunches with log reads, saves, optimizer, samplers, config, + placement, incidents, and GPU duty by role over the run's span (gpu_busy, percent). No verdict; reason from the facts. + + A vLLM sampler keeps one record per group of samples it served and one per failed sample. In samplers, sends is the + run's sample count, failed and cancelled count its records, and median_s is the median time of the groups it served. + In goodput, rollout is trainer idle time while one of those groups was being served.""" + state, run = await run_of(run_id) + run_id = run["run_id"] + ctx = await facts.context(state, run) + if relaunched := facts.relaunches(ctx): + # The scheduler names a Workload's pod orw-, so the Workload's name matches both; older records hold the pod's. + names = sorted({lane["name"].removeprefix("orw-") for lane in ctx["lanes"]}) + try: + ctx["changes"] = await sources.audit_events(names, relaunched[0]["stopped_at"] - 300, relaunched[-1]["started_at"] + 60) + except Exception as exc: + ctx["changes"] = {"error": f"Audit log unavailable: {exc if isinstance(exc, LookupError) else type(exc).__name__}"} + found = {} + if ops := ctx["ops"]: + # Duty over the run's own span: averaging in the idle days after it ended would understate it. + end = min(state["observed_at"], facts.last_op_at(ctx) + 60) + duty = await query("gpu_busy", run_id, None, facts.iso(max(ops[0]["at"] - 60, end - 7 * 86400)), facts.iso(end), 1) + roles: dict[str, set[str]] = {} + for lane in ctx["lanes"]: + for uuid in lane["devices"]: + roles.setdefault(uuid, set()).add(lane["role"]) + busy: dict[str, list[float]] = {} + for series in (s for s in duty["series"] if s["points"]): + # A GPU that time-sliced roles shared counts under the roles together, e.g. sampler+trainer. + busy.setdefault("+".join(sorted(roles.get(series["labels"].get("UUID")) or {"other"})), []).append(series["points"][-1][1]) + if busy: + found["gpu_busy"] = {role: round(sum(v) / len(v), 1) for role, v in busy.items()} + # On a shared runtime these are the runtime's GPUs, busy with every run it serves. + found["gpu_busy_window"] = {k: duty[k] for k in ("since", "until")} | {"scope": "runtime" if run["shared_runtime"] else "run"} + return {"run_id": run_id, **facts.summary(ctx), **found} + + +@router.get("/api/v1/inspect/sessions/{session_id}/summary", response_model=Doc) +async def session_summary(session_id: str): + """The session's record, whether its registry key is live, its SDK errors, and a line per model over the session's span.""" + store = get_state_store() + if (record := json.loads(await store.get_value(key("session_record", session_id)) or "null")) is None: + raise HTTPException(404, f"No session {session_id}") + live = await store.get_value(key("session", session_id)) is not None + # A model's ops belong to this session while it was alive, which ends when its live key expires after last_seen_at. + low, high = record["created_at"], math.inf if live else record["last_seen_at"] + 120 + state, models = await fleet.snapshot(), [] + for run in (r for r in state["runs"] if r["run_id"] in record["model_ids"]): + models.append({"model_id": run["run_id"], **facts.line(await facts.context(state, run, low, high))}) + errors = await store.read_samples(key("sdk_errors", session_id)) + return {**record, "live": live, "models": models} | ({"sdk_errors": {"count": len(errors), "last": errors[-1].get("message")}} if errors else {}) + + +@router.get("/api/v1/inspect/logs", response_model=Doc) +async def logs( + run_id: str, + pod: Annotated[str | None, Query(description="One of the run's pods, or 'client' for its client's lines")] = None, + q: str = Query("", max_length=1024), + severity: str | None = Query(None, pattern="^(DEBUG|INFO|WARNING|ERROR|CRITICAL|UNKNOWN)$"), + since: Bound = None, + until: Bound = None, + limit: int = Query(200, ge=1, le=1000), + cursor: Annotated[str | None, Query(max_length=16384, description="next_cursor of a page with the same filters and window")] = None, +): + """Newest first: the run's worker pods by owner label from Cloud Logging, and with pod=client its client's logs.log. That + file's lines carry no time, so since and until do not apply to it and its cursor is a byte offset.""" + since, until, low, high = span(since, until) + await sources.discover() + state, run = await run_of(run_id) + run_id = run["run_id"] + pods = {lane["pod"]: {"pod": lane["pod"], "node": lane["node"], "role": lane["role"]} for lane in fleet.lanes(state, run_id) if lane["pod"]} + result = {"run_id": run_id, "source": "gke", "since": since, "until": until, "sources": list(pods.values()), "records": [], "next_cursor": None} + if directory := await client_directory(run): + result["sources"].append({"pod": sources.CLIENT, "node": None, "role": sources.CLIENT}) + if directory and pod == sources.CLIENT: + if cursor and not cursor.isdigit(): + raise HTTPException(400, "Invalid or expired log cursor; repeat the query without it") + result["source"] = path = f"{directory}/logs.log" + try: + page = await asyncio.to_thread(client_runs.log_page, directory, int(cursor) if cursor else None, limit, q, severity) + return {**result, **page, "available": True} + except OSError as exc: + return {**result, "available": False, "error": f"{path} unreadable: {type(exc).__name__}"} + try: + return {**result, **await sources.log_page(run, since, until, cursor=cursor, limit=limit, q=q, pod=pod, severity=severity), "available": True} + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 400 and cursor: + raise HTTPException(400, "Invalid or expired log cursor; repeat the query without it") from None + return {**result, "available": False, "error": f"Cloud Logging refused the read (HTTP {exc.response.status_code}); check credentials and IAM"} + except Exception as exc: + return {**result, "available": False, "error": str(exc) if isinstance(exc, LookupError) else "Cloud Logging unavailable"} diff --git a/src/server/dashboard/sources.py b/src/server/dashboard/sources.py new file mode 100644 index 000000000..32a041a4e --- /dev/null +++ b/src/server/dashboard/sources.py @@ -0,0 +1,327 @@ +"""What the dashboard reads outside Redis: Kubernetes, Prometheus (OPEN_RL_PROMETHEUS_URL, else GKE's) and Cloud Logging.""" + +import asyncio +import json +import os +import re +import threading +import time +from datetime import UTC, datetime, timedelta +from functools import cache + +import google.auth +import google.oauth2.credentials +import httpx +from google.auth.transport.requests import Request + +from server.telemetry import epoch +from server.worker_manager import owner_id + +client: httpx.AsyncClient | None = None +METADATA = {"project": "project/project-id", "cluster": "instance/attributes/cluster-name", "location": "instance/attributes/cluster-location"} +detected: dict = {} +discovered_at = -60.0 +credentials = None +credentials_lock = threading.Lock() + + +async def send(method: str, url: str, **kwargs) -> httpx.Response: + global client + client = client or httpx.AsyncClient(timeout=10) + return (await client.request(method, url, **kwargs)).raise_for_status() + + +def namespace() -> str: + return os.getenv("OPEN_RL_WORKER_NAMESPACE") or os.getenv("POD_NAMESPACE") or "openrl-system" + + +def configuration() -> dict: + """OPEN_RL_GKE_PROJECT, _CLUSTER and _LOCATION, else what the metadata server said; OPEN_RL_GKE_TELEMETRY=0 turns GKE off.""" + fields = {key: os.getenv(f"OPEN_RL_GKE_{key.upper()}") or detected.get(key, "") for key in METADATA} + off = os.getenv("OPEN_RL_GKE_TELEMETRY", "auto").strip().lower() not in ("auto", "1", "true") + return {**fields, "configured": not off and all(fields.values())} + + +async def discover() -> None: + """In a cluster, read the GKE identity from the metadata server; retry a failed read after a minute.""" + global detected, discovered_at + if configuration()["configured"] or not os.getenv("KUBERNETES_SERVICE_HOST") or time.monotonic() - discovered_at < 60: + return + discovered_at = time.monotonic() + async with httpx.AsyncClient(timeout=1, trust_env=False, follow_redirects=False) as metadata: + + async def read(path: str) -> str: + response = await metadata.get(f"http://metadata.google.internal/computeMetadata/v1/{path}", headers={"Metadata-Flavor": "Google"}) + if response.status_code != 200 or response.headers.get("Metadata-Flavor") != "Google" or not 0 < len(response.text.strip()) <= 256: + raise ValueError("Invalid metadata response") + return response.text.strip() + + values = await asyncio.gather(*(read(path) for path in METADATA.values()), return_exceptions=True) + if not any(isinstance(value, Exception) for value in values): # never partial: plain GCE also exposes a project + detected = dict(zip(METADATA, values)) + + +def access_token() -> str: + global credentials + with credentials_lock: + if credentials is None: + credentials, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/cloud-platform.read-only"]) + if not credentials.valid: + credentials.refresh(Request()) + return credentials.token + + +async def gcp_headers() -> dict[str, str]: + # A hung token refresh holds the lock; bound it rather than start a thread per call. + headers = {"Authorization": f"Bearer {await asyncio.wait_for(asyncio.to_thread(access_token), timeout=10)}"} + # A person's gcloud login draws on gcloud's shared quota and fails with 429 unless it bills this project. + if isinstance(credentials, google.oauth2.credentials.Credentials) and not credentials.quota_project_id: + headers["x-goog-user-project"] = configuration()["project"] + return headers + + +def problem(status: dict) -> str | None: + """The one line that explains a pod that is not simply running. A container running again after a restart is not one.""" + if status.get("phase") == "Failed": + return f"Failed: {': '.join(p for p in (status.get('reason'), status.get('message')) if p) or 'see logs'}" + for cs in status.get("containerStatuses") or []: + waiting, ended = cs["state"].get("waiting"), cs["state"].get("terminated") + if waiting and waiting.get("reason") not in (None, "ContainerCreating", "PodInitializing"): + return f"{waiting['reason']}: {waiting.get('message') or ''}".strip(": ") + if ended and (ended.get("exitCode") or ended.get("reason") not in (None, "Completed")): + return f"{ended.get('reason') or 'Terminated'}: exit code {ended.get('exitCode')}" + (f" — {ended['message']}" if ended.get("message") else "") + if status.get("phase") == "Pending": + unscheduled = [c for c in status.get("conditions") or [] if c["type"] == "PodScheduled" and c["status"] != "True"] + return f"Unschedulable: {unscheduled[0].get('message') or 'no node available'}" if unscheduled else "Pending" + return None + + +def incidents(pod: str, statuses: list[dict]) -> list[dict]: + """The last termination of each container that restarted: when, why, and the exit code.""" + ended = [(cs["name"], t) for cs in statuses if (t := (cs.get("lastState") or {}).get("terminated")) and t.get("finishedAt")] + return [ + {"at": epoch(t["finishedAt"]), "pod": pod, "reason": t.get("reason") or "Terminated", "message": f"{name} exited with code {t.get('exitCode')}"} + for name, t in ended + ] + + +@cache +def kubernetes_api() -> tuple: + """The Kubernetes clients, built once per process from the in-cluster service account, else the kubeconfig. A failed + build is not cached, so the next read tries again.""" + from kubernetes import client, config # the cluster extra installs it + + try: + config.load_incluster_config() + except Exception: + config.load_kube_config() + return client.CoreV1Api(), client.CustomObjectsApi() + + +async def kubernetes() -> dict: + """Pods, nodes and scheduler Workloads, one plain list each; workloads is None when the scheduler cannot be read or is + not installed.""" + try: + from server.scheduler_worker_manager import age_seconds, worker_state + + core, custom = kubernetes_api() + except Exception as exc: + return {"pods": [], "nodes": [], "workloads": None, "errors": [f"Kubernetes unavailable: {exc}"]} + ns, errors = namespace(), [] + + async def items(name: str, call, *args) -> list[dict] | None: + try: + # Read as raw JSON, because the client's own decoding of a large pod list costs more than the request. + found = await asyncio.to_thread(call, *args, _preload_content=False, _request_timeout=6) + return json.loads(found.data)["items"] + except Exception as exc: + if getattr(exc, "status", None) != 404: # no scheduler CRD on a local stack + errors.append(f"{name} read failed: {exc}") + return None + + pods, nodes, workloads = await asyncio.gather( + items("pods", core.list_namespaced_pod, ns), + items("nodes", core.list_node), + items("workloads", custom.list_namespaced_custom_object, "openrl.io", "v1alpha1", ns, "workloads"), + ) + result, now = {"pods": [], "nodes": [], "workloads": None if workloads is None else [], "errors": errors}, time.time() + for pod in pods or []: + name, labels, statuses = pod["metadata"]["name"], pod["metadata"].get("labels") or {}, pod["status"].get("containerStatuses") or [] + row = {"name": name, "node": pod["spec"].get("nodeName"), "owner": labels.get("timeslice.io/owner"), "problem": problem(pod["status"])} + result["pods"].append(row | {"incidents": incidents(name, statuses), "restarts": sum(cs.get("restartCount") or 0 for cs in statuses)}) + for node in nodes or []: + labels, status = node["metadata"].get("labels") or {}, node["status"] + row = {"name": node["metadata"]["name"], "ready": any(c["type"] == "Ready" and c["status"] == "True" for c in status.get("conditions") or [])} + row.update(accelerator=labels.get("cloud.google.com/gke-accelerator") or labels.get("nvidia.com/gpu.product")) + row["gpu_capacity"] = int((status.get("capacity") or {}).get("nvidia.com/gpu", 0)) + result["nodes"].append(row | {"openrl_enabled": labels.get("openrl.io/enabled") == "true"}) # the scheduler places only on enabled nodes + for item in workloads or []: + spec, status, meta = item["spec"], item.get("status") or {}, item["metadata"] + deleting = bool(meta.get("deletionTimestamp")) + state, _ = worker_state(phase=status.get("phase") or "Pending", reason=status.get("reason") or "", node=status.get("nodeName"), deleting=deleting) + placed = next((c for c in status.get("conditions") or [] if c.get("type") == "Placed"), {}) + row = {"name": meta["name"], "role": spec["role"], "model_id": spec["modelID"], "training_kind": spec.get("trainingKind"), "state": state} + row.update(node=status.get("nodeName"), age_seconds=age_seconds(meta.get("creationTimestamp"), now)) + result["workloads"].append(row | {"message": placed.get("message") or status.get("reason")}) + return result + + +def scope() -> str: + return f",cluster={json.dumps(configuration()['cluster'])}" if configuration()["cluster"] else "" + + +def any_of(values: list[str]) -> str: + """A PromQL regex matcher for exactly these label values.""" + return json.dumps("|".join(re.escape(v) for v in values)) + + +def prometheus_source() -> str | None: + return "prometheus" if os.getenv("OPEN_RL_PROMETHEUS_URL") else "gke" if configuration()["configured"] else None + + +def step_seconds(start: float, end: float, points: int) -> int: + """The seconds between the points prom returns for [start, end]. DCGM is read and scraped every 10 s; a finer step only + repeats samples.""" + return max(10, int((end - start) / points)) + + +async def prom(query: str, start: float, end: float, points: int = 1) -> list[dict]: + """Prometheus matrix rows; start == end evaluates once, at end.""" + url, headers = os.getenv("OPEN_RL_PROMETHEUS_URL", "").rstrip("/"), {} + if not url: + if not configuration()["configured"]: + raise LookupError("No GPU metrics source: not on GKE and OPEN_RL_PROMETHEUS_URL is unset") + url, headers = f"https://monitoring.googleapis.com/v1/projects/{configuration()['project']}/location/global/prometheus", await gcp_headers() + step = step_seconds(start, end, points) + # Samples land on multiples of step, so queries made at different moments share timestamps. + start -= start % step if start < end else 0 + payload = ( + await send("GET", f"{url}/api/v1/query_range", params={"query": query, "start": start, "end": end, "step": step}, headers=headers) + ).json() + if payload.get("status") != "success": + raise ValueError(payload.get("error") or "query failed") + return payload["data"]["result"] + + +# GKE copies each pod label onto its log entries as k8s-pod/, with the dots in the key as underscores. +OWNER = 'labels."k8s-pod/timeslice_io/owner"' +# An in-cluster client labels its own lines with its run and this source, so they read with the run's worker logs. +RUN_LABEL = "openrl_run_id" +SOURCE_LABEL = "openrl_source" +CLIENT = "client" +TELEMETRY = "telemetry" # telemetry's own event lines, kept as history and never read here +# GKE files every stderr line as ERROR, though most are progress bars, INFO logs and warnings; such a line takes the level it prints. +RELABELED = ("DEBUG", "INFO", "WARNING") +LEVEL, PROGRESS = re.compile(r"\b(DEBUG|INFO|WARNING|WARN|ERROR|CRITICAL)\b"), re.compile(r"\r|\d+%(\||\s+Completed)") + + +def logs_filter(run: dict, start: str, end: str, severity: str | None, q: str, pod: str | None = None) -> str: + """The run's worker pods by the scheduler's owner label and its client's lines in the window, or one of them.""" + config = configuration() + labels = {"project_id": config["project"], "location": config["location"], "cluster_name": config["cluster"], "namespace_name": namespace()} + clauses = ['resource.type="k8s_container"', *(f"resource.labels.{label}={json.dumps(value)}" for label, value in labels.items())] + client = f"(labels.{RUN_LABEL}={json.dumps(run['run_id'])} AND labels.{SOURCE_LABEL}={json.dumps(CLIENT)})" + owner = f"{OWNER}={json.dumps(owner_id(run['runtime_id']).rstrip('-'))}" # the scheduler trims a truncated label value as well + if pod and pod != CLIENT: + # One of the run's pods, never any pod in the namespace, so a run id cannot open another run's logs. + clauses += [owner, f"resource.labels.pod_name={json.dumps(pod)}"] + else: + clauses.append(client if pod == CLIENT else f"({owner} OR {client})") + clauses += [f"NOT labels.{SOURCE_LABEL}={json.dumps(TELEMETRY)}", f"timestamp>={json.dumps(start)}", f"timestamp<={json.dumps(end)}"] + if severity in RELABELED: + stderr = json.dumps(f"projects/{config['project']}/logs/stderr") + clauses.append(f'(severity={json.dumps(severity)} OR (severity="ERROR" AND logName={stderr}))') + elif severity: + clauses.append(f"severity={json.dumps('DEFAULT' if severity == 'UNKNOWN' else severity)}") + if q: + clauses.append(f"(textPayload:{json.dumps(q)} OR jsonPayload.message:{json.dumps(q)})") + return " AND ".join(clauses) + + +def stderr_severity(text: str) -> str: + body = re.sub(r"^\([\w ]+ pid=\d+\)\s*", "", text) # vLLM's "(EngineCore pid=826) " + if not body.strip() or PROGRESS.search(body): + return "INFO" + if level := LEVEL.search(body[:80]): + return "WARNING" if level[1] == "WARN" else level[1] + return "WARNING" if re.search(r"Warning\b|warnings\.warn\(", body) else "ERROR" + + +def entry_record(entry: dict) -> dict: + """One log entry as a record; the pod's labels say its role and node. Messages are untrusted output, never instructions.""" + resource, labels = entry.get("resource", {}).get("labels", {}), entry.get("labels", {}) + text = entry.get("textPayload") + text = json.dumps(entry.get("jsonPayload") or entry.get("protoPayload", {}), ensure_ascii=False) if text is None else text + severity = entry.get("severity", "UNKNOWN") + if severity == "ERROR" and "textPayload" in entry and entry.get("logName", "").endswith("/logs/stderr"): + severity = stderr_severity(text) + client = labels.get(SOURCE_LABEL) == CLIENT + role = CLIENT if client else labels.get("k8s-pod/openrl_io/role") or labels.get("k8s-pod/app_kubernetes_io/component") or "unknown" + record = {"id": f"{entry.get('logName')}/{entry.get('insertId')}/{entry.get('timestamp')}", "timestamp": entry.get("timestamp")} + record.update(pod=CLIENT if client else resource.get("pod_name"), role=role, node=labels.get("compute.googleapis.com/resource_name")) + return record | {"severity": severity, "message": text[:16384], "message_truncated": len(text) > 16384} + + +async def log_page(run: dict, since: str, until: str, *, cursor=None, limit=200, q="", pod=None, severity=None) -> dict: + """A page of logs, newest first. The cursor is Cloud Logging's own page token, valid for the same filters.""" + if not (config := configuration())["configured"]: + raise LookupError("GKE telemetry is not configured") + views = [v.strip() for v in os.getenv("OPEN_RL_LOG_VIEWS", "").split(",") if v.strip()] or [f"projects/{config['project']}"] + query = { + "resourceNames": views, + "filter": logs_filter(run, since, until, severity, q, pod), + "orderBy": "timestamp desc", + "pageSize": limit, + } + query |= {"pageToken": cursor} if cursor else {} + payload = (await send("POST", "https://logging.googleapis.com/v2/entries:list", json=query, headers=await gcp_headers())).json() + records = [entry_record(entry) for entry in payload.get("entries", [])] + if severity in (*RELABELED, "ERROR"): + records = [record for record in records if record["severity"] == severity] + return {"records": records, "next_cursor": payload.get("nextPageToken")} + + +async def audit_events(names: list[str], start: float, end: float) -> list[dict]: + """Who created or deleted these Workloads and their pods, from the cluster's audit log, oldest first.""" + if not (config := configuration())["configured"]: + raise LookupError("GKE telemetry is not configured") + audit_log = f"projects/{config['project']}/logs/cloudaudit.googleapis.com%2Factivity" + methods = ("io.openrl.v1alpha1.workloads.create", "io.openrl.v1alpha1.workloads.delete", "io.k8s.core.v1.pods.delete") + clauses = [ + f"logName={json.dumps(audit_log)}", + f"resource.labels.cluster_name={json.dumps(config['cluster'])}", + "(" + " OR ".join(f"protoPayload.resourceName:{json.dumps(name)}" for name in names) + ")", + "(" + " OR ".join(f"protoPayload.methodName={json.dumps(m)}" for m in methods) + ")", + f"timestamp>={json.dumps(datetime.fromtimestamp(start, UTC).isoformat())}", + f"timestamp<={json.dumps(datetime.fromtimestamp(end, UTC).isoformat())}", + ] + query = {"resourceNames": [f"projects/{config['project']}"], "filter": " AND ".join(clauses), "orderBy": "timestamp asc", "pageSize": 50} + payload = (await send("POST", "https://logging.googleapis.com/v2/entries:list", json=query, headers=await gcp_headers())).json() + return [ + { + "at": e.get("timestamp"), + "action": e["protoPayload"]["methodName"].rsplit(".", 2)[-2] + " " + e["protoPayload"]["methodName"].rsplit(".", 1)[-1], + "resource": e["protoPayload"].get("resourceName", "").rsplit("/", 1)[-1], + "by": e["protoPayload"].get("authenticationInfo", {}).get("principalEmail"), + } + for e in payload.get("entries", []) + ] + + +def window(since: str | None = None, until: str | None = None) -> tuple[str, str, float, float]: + """(since, until) in ISO-8601 UTC, then in Unix seconds; either form is accepted. until defaults to now, since to 30 minutes before it or a + span before it such as 1h; at most seven days.""" + + def parse(value: str) -> datetime: + if re.fullmatch(r"\d+(\.\d+)?", value): + return datetime.fromtimestamp(float(value), UTC) + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return (parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)).astimezone(UTC) + + end = parse(until) if until else datetime.now(UTC) + if ago := re.fullmatch(r"(\d+)([mhd])", since or ""): + since = (end - timedelta(**{{"m": "minutes", "h": "hours", "d": "days"}[ago[2]]: int(ago[1])})).isoformat() + start = parse(since) if since else end - timedelta(minutes=30) + if start >= end or end - start > timedelta(days=7): + raise ValueError("Time range must be positive and no longer than seven days") + return start.isoformat(timespec="microseconds"), end.isoformat(timespec="microseconds"), start.timestamp(), end.timestamp() diff --git a/src/server/dashboard/static/index.html b/src/server/dashboard/static/index.html new file mode 100644 index 000000000..e5b5b63bf --- /dev/null +++ b/src/server/dashboard/static/index.html @@ -0,0 +1,12 @@ + + + + + + + OpenRL · Cluster + + + +
+ diff --git a/src/server/dashboard/static/style.css b/src/server/dashboard/static/style.css new file mode 100644 index 000000000..99a0ee4ca --- /dev/null +++ b/src/server/dashboard/static/style.css @@ -0,0 +1,351 @@ +:root { color-scheme: light dark; /* A scrollbar appearing as content loads would otherwise narrow and reflow the page. */ scrollbar-gutter: stable; } +body { margin: 0; background: light-dark(#ffffff, #131519); } +#app { --ink: light-dark(#202328, #e6e8ed); --dim: light-dark(#626874, #a0a6b1); --ground: light-dark(#ffffff, #131519); --paper: light-dark(#f7f8fa, #191c21); --line: light-dark(#e3e6eb, #2a2e35); --hover: light-dark(#eff1f4, #20242b); --selected: light-dark(#e8ebf0, #282d35); --control-border: light-dark(#c9ced7, #424954); --accent: light-dark(#416bc3, #83a9ff); --hue-0: var(--accent); --hue-1: light-dark(#357e70, #76b3a2); --hue-2: light-dark(#7a60bb, #ad98dc); --hue-3: light-dark(#a87434, #c3a072); --hue-4: light-dark(#a8496b, #cd8ca5); --hue-5: light-dark(#2f8a9a, #7cb3bd); --hue-6: light-dark(#858533, #b5b579); --hue-7: light-dark(#b85538, #e39a7f); --status-success: light-dark(#27725c, #83baa5); --status-warning: light-dark(#895e1e, #ccb079); --status-error: light-dark(#ab4249, #da949a); --status-complete: light-dark(#496a97, #9bb0ce); --radius: 5px; --gutter: 22px; --node-label-width: 200px; --gpu-index-width: 38px; --gpu-index-gap: 6px; --sans: Inter, ui-sans-serif, system-ui, sans-serif; --mono: "SFMono-Regular", Consolas, monospace; color: var(--ink); background: var(--ground); width: 100%; min-height: 100dvh; font: 13px/1.5 var(--sans); -webkit-font-smoothing: antialiased; } +#app * { box-sizing: border-box; } +#app :is(h1, h2, p, figure) { margin: 0; } +#app a { color: inherit; text-decoration: none; } +#app a:hover { text-decoration: underline; text-underline-offset: 3px; } +#app button { cursor: pointer; font: inherit; color: inherit; } +#app :where(input, select, .chip, .time-shift) { min-height: 32px; max-width: 100%; padding: 5px 9px; border: 1px solid var(--line); border-radius: var(--radius); background-color: var(--paper); color: var(--ink); font: inherit; line-height: 20px; box-shadow: none; } +#app select { appearance: none; padding-right: 30px; background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='8' viewBox='0 0 12 8'%3E%3Cpath d='m1 1.5 5 5 5-5' fill='none' stroke='%23878d98' stroke-width='1.4' stroke-linecap='round' stroke-linejoin='round'/%3E%3C/svg%3E"); background-repeat: no-repeat; background-position: right 10px center; } +#app input::placeholder { color: var(--dim); opacity: 1; } +#app :is(input, select):hover, #app .chip:hover:not(:disabled) { border-color: var(--control-border); } +#app .chip:hover:not(:disabled) { background-color: var(--hover); } +#app .chip[aria-pressed="true"] { background: var(--selected); border-color: var(--control-border); } +#app button:disabled { cursor: default; opacity: 0.5; } +#app :is(a, button, input, select):focus-visible { outline: 1px solid var(--accent); outline-offset: 3px; } +#app :is(input, select):focus { border-color: var(--control-border); box-shadow: none; } +#app :is(input, select):focus:not(:focus-visible) { outline: none; } +#app .appbar { display: flex; flex-wrap: wrap; align-items: center; gap: 12px 28px; padding: 14px var(--gutter); border-bottom: 1px solid var(--line); } +#app nav { display: flex; flex-wrap: wrap; gap: 20px; } +#app nav a { display: inline-flex; align-items: center; min-height: 32px; color: var(--dim); } +#app nav a:hover { color: var(--ink); text-decoration: none; } +#app nav [aria-current="page"] { color: var(--ink); text-decoration: underline; text-decoration-thickness: 2px; text-underline-offset: 8px; } +#app .brand { font-size: 16px; font-weight: 500; } +#app #connection { margin-left: auto; color: var(--dim); font-size: 12px; } +#app .pad { padding: 22px var(--gutter); } +#app .heading { margin-bottom: 16px; font-size: 22px; font-weight: 500; line-height: 1.3; overflow-wrap: anywhere; } +#app .mono { font-family: var(--mono); } +#app .muted { color: var(--dim); } +#app .micro { font-size: 11px; } +#app .empty-state { padding: 16px 0; color: var(--dim); } +/* Jobs, processes and health share the same table rhythm. */ +#app :is(.job-list-head, .job-list-row) { display: grid; grid-template-columns: minmax(0, 1fr) 112px 100px 112px 100px; align-items: center; gap: 16px; padding: 12px 0; } +#app .job-list-head { color: var(--dim); font-size: 12px; } +#app .job-list-row { border-top: 1px solid var(--line); } +#app .job-list-row > * { min-width: 0; overflow-wrap: anywhere; } +#app .job-list-row:hover { text-decoration: none; } +#app .job-list-row:hover .job-identity > span:first-child { text-decoration: underline; text-underline-offset: 3px; } +#app .job-identity { display: grid; gap: 3px; min-width: 0; overflow-wrap: anywhere; } +#app .job-identity .micro { font-size: 12px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +#app :is(.job-list-head, .job-list-row) > :nth-child(n + 4) { text-align: right; font-variant-numeric: tabular-nums; } +/* Overview: search and status chips above the run list. */ +#app .overview-toolbar { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 16px; margin-bottom: 14px; } +#app #run-search { flex: 1 1 280px; max-width: 420px; } +#app .run-filters { display: flex; flex-wrap: wrap; gap: 6px; } +#app .metric-filters { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin-bottom: 12px; } #app .metric-filters input { width: 28ch; } +#app .run-filter-count { margin-left: 4px; color: var(--dim); font-variant-numeric: tabular-nums; } +#app :is(.job-list-head, .job-list-row).run-row { position: relative; grid-template-columns: minmax(0, 1fr) 120px 72px 88px 160px; gap: 24px; } +/* The run link covers its row. */ +#app .run-row .job-identity a::after { content: ""; position: absolute; inset: 0; } +#app .run-row:hover .job-identity a { text-decoration: underline; text-underline-offset: 3px; } +#app :is(.job-list-head, .job-list-row).run-row > :is(:nth-child(3), :nth-child(4)) { text-align: right; font-variant-numeric: tabular-nums; } +#app .run-row > .run-session, #app .job-list-head.run-row > :nth-child(5) { text-align: left; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +/* Sessions group the run rows; a session's runs are indented under its header. */ +#app .state { display: inline-flex; align-items: center; gap: 7px; white-space: nowrap; } +#app :is(.state-dot, .health-dot) { width: 6px; height: 6px; border-radius: 50%; background: currentColor; flex: none; } +#app .state-running { color: var(--status-success); } +#app .state-pending { color: var(--status-warning); } +#app .state-failed { color: var(--status-error); } +#app .state-completed { color: var(--status-complete); } +#app .state-neutral { color: var(--dim); } +#app :is(.run-table, .scheduler-table) { width: 100%; border-collapse: collapse; } +#app :is(.run-table, .scheduler-table) :is(th, td) { padding: 12px 12px 12px 0; border-bottom: 1px solid var(--line); text-align: left; vertical-align: top; overflow-wrap: anywhere; } +#app :is(.run-table, .scheduler-table) :is(th, td):last-child { padding-right: 0; } +#app :is(.run-table, .scheduler-table) th { color: var(--dim); font-weight: 500; } +#app .run-table { table-layout: fixed; min-width: 720px; } +#app .run-table th:first-child { width: 38%; } +#app .run-table :is(th, td):last-child { text-align: right; } +#app .health-status { --health-tone: var(--status-warning); display: inline-flex; align-items: center; gap: 8px; white-space: nowrap; padding: 4px 8px; border-radius: var(--radius); color: var(--health-tone); background: color-mix(in srgb, var(--health-tone) 9%, var(--ground)); } +#app .health-error { --health-tone: var(--status-error); } +#app .health-success { --health-tone: var(--status-success); } +#app .health-message { display: flex; align-items: baseline; flex-wrap: wrap; gap: 8px 12px; margin: 12px 0; overflow-wrap: anywhere; } +/* Nodes: the header, GPU lanes and duty values use one column grid. */ +#app .nodes-heading { display: flex; justify-content: space-between; align-items: center; flex-wrap: wrap; gap: 12px 16px; margin-bottom: 18px; } +#app :is(.nodes-heading, .run-heading) .heading { margin: 0; } +#app .time-control { position: relative; display: grid; grid-template-columns: 32px minmax(0, 1fr) 32px 80px; width: 424px; max-width: 100%; align-items: center; gap: 8px; color: var(--dim); font-size: 12px; } +#app .time-picker { min-width: 0; color: var(--ink); } +#app .time-summary { display: flex; align-items: center; justify-content: space-between; gap: 8px; height: 32px; padding: 5px 10px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--paper); list-style: none; cursor: pointer; line-height: 20px; white-space: nowrap; } +#app .time-summary::-webkit-details-marker { display: none; } +#app .time-summary > span { overflow: hidden; text-overflow: ellipsis; } +#app .time-chevron { display: block; flex: none; width: 16px; height: 16px; fill: none; stroke: var(--dim); stroke-width: 1.5; stroke-linecap: round; stroke-linejoin: round; } +#app .time-summary:hover, #app .time-picker[open] .time-summary { border-color: var(--control-border); } +#app .time-summary:focus-visible { outline: 1px solid var(--accent); outline-offset: 3px; } +#app .time-shift { display: grid; place-items: center; width: 32px; height: 32px; padding: 0; line-height: 20px; } +#app .time-popover { position: absolute; z-index: 5; top: 38px; left: 0; right: 88px; display: grid; gap: 12px; padding: 14px; border: 1px solid var(--control-border); border-radius: var(--radius); background: var(--ground); } +#app .time-popover label { display: grid; align-items: center; grid-template-columns: 70px minmax(0, 1fr); gap: 8px; } +#app .time-popover :is(input, select) { width: 100%; height: 32px; } +#app .time-control input { min-width: 0; } +#app :is(.node-lane, .node-time-header) { display: grid; grid-template-columns: var(--node-label-width) minmax(0, 1fr) 48px; align-items: start; gap: 12px; } +#app .node-lane { padding: 10px 0; border-bottom: 1px solid var(--line); } +#app .node-time-header { padding-bottom: 8px; color: var(--dim); font-size: 11px; } +#app .node-axis { position: relative; display: flex; justify-content: space-between; margin-left: calc(var(--gpu-index-width) + var(--gpu-index-gap)); font-variant-numeric: tabular-nums; } +#app .node-axis span:nth-child(2) { position: absolute; left: 33.333333%; } +#app .node-axis span:nth-child(3) { position: absolute; left: 66.666667%; } +#app .node-axis span:not(:first-child):not(:last-child) { transform: translateX(-50%); } +#app .node-duty { text-align: right; font-variant-numeric: tabular-nums; line-height: 26px; } +#app .node-time-header .node-duty { line-height: inherit; } +#app .node-lane-label { display: flex; align-items: center; gap: 7px; min-width: 0; min-height: 26px; padding: 0; border: 0; background: transparent; text-align: left; white-space: nowrap; } +#app .node-lane-label:hover .node-accelerator { text-decoration: underline; text-underline-offset: 3px; } +#app .node-accelerator { min-width: 0; overflow: hidden; text-overflow: ellipsis; } +#app .unavailable-selection { display: flex; justify-content: space-between; align-items: center; gap: 12px; margin-bottom: 12px; color: var(--status-warning); } +#app .claim-label { flex: none; padding: 1px 5px; border-radius: 3px; background: var(--paper); color: var(--dim); font-size: 11px; line-height: 18px; white-space: nowrap; } +#app .gpu-capacity { display: grid; grid-template-columns: var(--gpu-index-width) minmax(0, 1fr); gap: var(--gpu-index-gap); min-width: 0; } +#app .gpu-lane-ids { display: grid; align-items: center; color: var(--dim); font: 11px var(--mono); } +#app .gpu-lane-ids span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +#app .capacity-track { position: relative; min-width: 0; background: repeating-linear-gradient( to bottom, transparent 0 calc(var(--gpu-lane-height) - 1px), color-mix(in srgb, var(--line) 70%, transparent) calc(var(--gpu-lane-height) - 1px) var(--gpu-lane-height) ); } +#app :is(.capacity-track, .node-axis, .activity-track, .activity-axis, .allocation-expansion .chart-plot svg, .allocation-expansion .chart-empty) { touch-action: pan-y; cursor: grab; } +#app.node-navigating :is(.capacity-track, .node-axis, .activity-track, .activity-axis, .allocation-expansion .chart-plot svg, .allocation-expansion .chart-empty) { cursor: grabbing; user-select: none; } +#app .capacity-track::before { content: ""; position: absolute; inset: 0; pointer-events: none; background: repeating-linear-gradient( to right, transparent 0 calc(33.333333% - 1px), color-mix(in srgb, var(--line) 45%, transparent) calc(33.333333% - 1px) 33.333333% ); } +#app .hue-0 { --allocation-color: var(--hue-0); } +#app .hue-1 { --allocation-color: var(--hue-1); } +#app .hue-2 { --allocation-color: var(--hue-2); } +#app .hue-3 { --allocation-color: var(--hue-3); } +#app .hue-4 { --allocation-color: var(--hue-4); } +#app .hue-5 { --allocation-color: var(--hue-5); } +#app .hue-6 { --allocation-color: var(--hue-6); } +#app .hue-7 { --allocation-color: var(--hue-7); } +#app .capacity-allocation { container: allocation / inline-size; position: absolute; overflow: hidden; display: flex; align-items: center; justify-content: space-between; gap: 8px; min-width: 0; padding: 0; border: 0; border-radius: 2px; background: color-mix( in srgb, var(--allocation-color, var(--accent)) 19%, var(--ground) ); text-align: left; } +#app .capacity-allocation:hover { background: color-mix( in srgb, var(--allocation-color, var(--accent)) 27%, var(--ground) ); } +#app .capacity-allocation.selected:not(.held) { z-index: 1; background: color-mix( in srgb, var(--allocation-color, var(--accent)) 27%, var(--ground) ); outline: 1px solid color-mix(in srgb, var(--allocation-color, var(--accent)) 70%, var(--line)); } +#app .capacity-allocation.ended { --allocation-color: var(--dim); color: var(--dim); } +#app .capacity-allocation.held { overflow: hidden; background: color-mix(in srgb, var(--line) 45%, var(--ground)); } +#app :is(.allocation-name, .allocation-count) { position: relative; z-index: 1; } +#app .capacity-allocation.unknown-mapping { position: relative; width: 100%; height: 26px; margin-block: 3px; } +#app .allocation-name { margin-left: 7px; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +#app .allocation-count { margin-right: 7px; flex: none; margin-left: auto; color: var(--dim); font-size: 11px; white-space: nowrap; } +@container allocation (max-width: 160px) { + #app :is(.allocation-name, .allocation-count) { display: none; } +} +#app .hold { position: absolute; top: 0; bottom: 0; background: color-mix(in srgb, var(--allocation-color) 35%, var(--ground)); pointer-events: auto; } +#app .hold:is(:hover, .selected) { background: color-mix(in srgb, var(--allocation-color) 65%, var(--ground)); } +#app .allocation-expansion { --activity-label-width: clamp(160px, 25vw, 240px); display: grid; grid-template-columns: minmax(0, 1fr); gap: 12px; margin: 0 calc(-1 * var(--gutter)); padding: 16px var(--gutter); border-bottom: 1px solid var(--line); } +#app .allocation-expansion > div { min-width: 0; } +#app .node-gpu-detail { display: grid; grid-template-columns: minmax(0, 1fr); gap: 12px; border-top: 1px solid var(--line); padding-top: 16px; } +#app .allocation-detail-head { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 12px; } +#app .allocation-detail-head h2 { font-size: 15px; font-weight: 500; line-height: 1.4; overflow-wrap: anywhere; } +#app .detail-close { display: grid; place-items: center; width: 28px; height: 28px; padding: 0; border: 1px solid transparent; border-radius: var(--radius); color: var(--dim); background: transparent; font-size: 22px; line-height: 1; } +#app .detail-close:hover { border-color: var(--line); background: var(--paper); color: var(--ink); } +#app .allocation-device-picker { display: flex; flex-wrap: wrap; gap: 4px; } +#app .allocation-device-picker .chip { border-color: transparent; background: transparent; color: var(--dim); } +#app .allocation-device-picker .chip:not([data-device="all"]) { min-width: 10ch; /* Fits "7 · 100%", so arriving and updating values never resize the chips. */ font-variant-numeric: tabular-nums; /* Equal-width digits: a new value of the same length doesn't move the text. */ text-align: center; } #app .allocation-device-picker .chip[aria-pressed="true"], #app .allocation-device-picker .chip:hover { border-color: var(--line); background: var(--paper); color: var(--ink); } #app .allocation-memory { color: var(--dim); font-size: 12px; } #app .allocation-memory strong { display: inline-block; min-width: 10ch; /* Fits "1536.0 GiB", so the label doesn't slide when the value arrives. */ margin-left: 8px; color: var(--ink); font-size: 15px; font-weight: 500; text-align: right; } +#app .allocation-detail-footer { display: flex; flex-wrap: wrap; justify-content: space-between; gap: 12px; color: var(--dim); font-size: 11px; } +#app :is(.activity-header, .activity-row) { display: grid; grid-template-columns: var(--activity-label-width) minmax(0, 1fr); column-gap: 12px; align-items: center; } +#app .activity-header { padding-block: 4px 10px; } +#app .activity-header h3 { margin: 0; font-size: 13px; font-weight: 500; } +#app .activity-axis { display: flex; justify-content: space-between; color: var(--dim); font: 11px var(--mono); } +#app .activity-row { min-height: 46px; border-top: 1px solid var(--line); } +#app .activity-label { display: flex; align-items: center; gap: 8px; min-width: 0; padding: 6px 8px; border: 1px solid transparent; border-radius: var(--radius); background: none; text-align: left; font-size: 12px; } +#app a.activity-label:hover { background: var(--hover); } +#app .activity-row[data-selected="true"] .activity-label { border-color: var(--line); background: var(--paper); } +#app .activity-swatch { flex: none; width: 9px; height: 9px; border-radius: 2px; background: color-mix(in srgb, var(--allocation-color) 65%, var(--ground)); } +#app .activity-label-text { min-width: 0; } +#app .activity-name { display: block; min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +#app .activity-meta { display: block; margin-top: 2px; color: var(--dim); font-size: 11px; } +#app .activity-track { position: relative; min-width: 0; height: 46px; display: flex; align-items: center; background: repeating-linear-gradient(to right, var(--line) 0 1px, transparent 1px 25%); box-shadow: inset -1px 0 var(--line); } +#app .activity-track svg { display: block; width: 100%; height: 24px; } +#app .activity-block { fill: color-mix(in srgb, var(--allocation-color) 35%, var(--ground)); cursor: default; } +#app .activity-block[data-placement] { cursor: pointer; } +#app .activity-block[data-selected="true"], #app .activity-block:hover { fill: color-mix(in srgb, var(--allocation-color) 65%, var(--ground)); } +#app .activity-block[data-selected="true"] { stroke: var(--allocation-color); stroke-width: 1px; } +/* A native SVG outline spans the gaps between intervals; stroke each interval instead. */ +#app .activity-block:focus { outline: none; } +#app .activity-block:focus-visible { stroke: var(--accent); stroke-width: 2px; } +/* Worker-recorded ops inside a GPU turn. The turn underneath stays visible + where no op was recorded. */ +#app [data-op] { --op-color: #2a9d8f; } +#app [data-op="forward_backward"] { --op-color: #3366cc; } +#app [data-op="forward"] { --op-color: #6fa8dc; } +#app [data-op="optim_step"] { --op-color: #8e44ad; } +#app [data-op="sample"] { --op-color: #2e9e5b; } +#app [data-op="weight_sync"] { --op-color: #e67e22; } +#app [data-op="wake_up"], #app [data-op="sleep"] { --op-color: #95a5a6; } +#app [data-op="init"] { --op-color: #5d6d7e; } +#app .activity-block.op { fill: var(--op-color); } +#app .activity-block.op:hover { fill: color-mix(in srgb, var(--op-color) 75%, var(--ink, #000)); } +#app .op-legend { display: flex; flex-wrap: wrap; gap: 4px 14px; margin: 8px 0 0 calc(var(--activity-label-width, 0px) + 12px); font-size: 12px; color: var(--muted, inherit); } +#app .op-legend > span { display: inline-flex; align-items: center; gap: 5px; } +#app .op-swatch { width: 10px; height: 10px; border-radius: 2px; background: var(--op-color); } +#app .op-swatch.turn { background: color-mix(in srgb, currentColor 25%, transparent); } +#app .cross-node-activity { --activity-label-width: clamp(160px, 25vw, 240px); border-top: 1px solid var(--line); padding-top: 16px; } +#app .cross-node-activity .activity-meta { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +/* The Run timeline has no pan gesture; a click opens the logs at that time. */ +#app section.cross-node-activity .activity-track { cursor: pointer; } +#app [data-crosshair] :is(.activity-track, .capacity-track)::after { content: ""; position: absolute; pointer-events: none; inset-block: 0; left: var(--crosshair); border-left: 1px dashed var(--dim); } +#app .activity-tooltip { position: fixed; z-index: 10; pointer-events: none; max-width: min(360px, calc(100vw - 16px)); max-height: calc(100dvh - 16px); overflow: hidden; overflow-wrap: anywhere; padding: 9px 12px; border: 1px solid var(--control-border); border-radius: var(--radius); background: var(--paper); font-size: 12px; } +#app .activity-tooltip-label { display: block; margin-bottom: 4px; } +#app :is(.activity-tooltip-time, .activity-tooltip-detail) { color: var(--dim); } +#app .activity-status { position: absolute; left: 8px; max-width: calc(100% - 16px); padding: 2px 4px; color: var(--dim); background: var(--ground); font-size: 11px; } +#app .activity-status.unavailable { color: var(--status-warning); } +/* Chart layout stays separate from the SVG gridlines. */ +#app div.chart-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 340px), 1fr)); gap: 24px 20px; margin-top: 16px; } +#app .chart { --metric-stroke: var(--dim); --plot-height: 160px; min-width: 0; min-height: calc(var(--plot-height) + 82px); } +#app .chart > .chart-empty { display: grid; place-items: center; height: var(--plot-height); margin: 0; } +#app .chart[data-tone="accent"] { --metric-stroke: color-mix(in srgb, var(--accent) 75%, var(--dim)); } +#app .allocation-expansion .chart { --plot-height: 100px; min-height: calc(var(--plot-height) + 82px); } +#app .allocation-expansion .chart-plot { grid-template-columns: var(--activity-label-width) minmax(0, 1fr); column-gap: 12px; } +#app .allocation-expansion .chart-empty { margin-left: calc(var(--activity-label-width) + 12px); } +#app .allocation-expansion .chart-line { stroke-width: 1.5; } +#app .chart-head { display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 5px 16px; margin-bottom: 12px; } +#app .chart-head h2 { font-size: 13px; font-weight: 500; overflow-wrap: anywhere; } +#app .chart-latest { display: flex; align-items: baseline; gap: 7px; color: var(--dim); font-size: 11px; white-space: nowrap; } +#app .chart-latest strong { color: var(--ink); font: 500 15px var(--mono); } +#app .chart-plot { position: relative; display: grid; grid-template-columns: 44px minmax(0, 1fr); grid-template-rows: var(--plot-height) auto; column-gap: 8px; min-width: 0; } +#app .chart-y { margin-block: -6px; display: flex; flex-direction: column; justify-content: space-between; text-align: right; color: var(--dim); font: 11px var(--mono); } +#app .chart-plot:focus-visible { outline: 1px solid var(--control-border); outline-offset: 4px; } +#app .chart-plot svg { display: block; width: 100%; height: 100%; overflow: visible; border-bottom: 1px solid var(--line); } +#app .chart-x { grid-column: 2; position: relative; display: flex; justify-content: space-between; padding-top: 8px; color: var(--dim); font: 11px var(--mono); } +#app svg .chart-grid { stroke: var(--line); stroke-width: 1; } +#app .chart-line { fill: none; stroke: var(--metric-stroke); stroke-width: 1.6; stroke-linejoin: round; stroke-linecap: round; } +#app .chart-area { fill: var(--metric-stroke); fill-opacity: 0.07; } +#app .chart-band { fill: var(--allocation-color); fill-opacity: 0.55; } +#app .activity-block.wait { fill: var(--dim); fill-opacity: 0.16; } +#app .op-swatch.wait { background: var(--dim); opacity: 0.3; } +#app .chart-band.sampler { fill-opacity: 0.28; } +#app .chart-band-swatch { display: inline-block; width: 9px; height: 9px; margin-right: 5px; border-radius: 2px; background: var(--allocation-color); opacity: 0.55; vertical-align: -1px; } +#app .chart-band-key.sampler .chart-band-swatch { opacity: 0.28; } +#app .chart-dot { stroke: var(--metric-stroke); stroke-width: 5; stroke-linecap: round; } +#app .chart-hover { width: 0; position: absolute; top: 0; height: var(--plot-height); pointer-events: none; z-index: 2; } +#app .chart-hover[hidden] { display: none; } +#app .chart-cursor { position: absolute; inset: 0 auto 0 0; border-left: 1px dashed var(--control-border); } +#app .chart-crosshair { stroke: var(--control-border); stroke-dasharray: 3 3; } +#app .chart-tip { position: absolute; top: 6px; left: 8px; padding: 5px 8px; border: 1px solid var(--control-border); border-radius: var(--radius); background: var(--paper); color: var(--ink); font: 11px/1.5 var(--mono); white-space: nowrap; } +#app .chart-point { position: absolute; width: 6px; height: 6px; border-radius: 50%; background: var(--metric-stroke); transform: translate(-50%, -50%); } +#app .chart-summary { display: flex; flex-wrap: wrap; gap: 5px 18px; margin-top: 10px; color: var(--dim); font-size: 11px; } +#app .chart-summary strong { margin-left: 4px; color: var(--ink); font-family: var(--mono); font-weight: 400; } +#app .chart-note { color: var(--status-warning); } +#app .chart-empty { min-height: var(--plot-height); display: flex; align-items: center; color: var(--dim); } +/* Run details and logs. */ +#app .overview-back { margin-bottom: 16px; color: var(--dim); } +#app .run-heading { display: flex; align-items: center; flex-wrap: wrap; gap: 8px 16px; } +#app .run-description { margin-top: 4px; color: var(--dim); overflow-wrap: anywhere; } +#app .run-state { margin-top: 12px; overflow-wrap: anywhere; } +#app .run-toolbar { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 8px 16px; margin: 16px 0; padding-bottom: 12px; border-bottom: 1px solid var(--line); } +#app .run-time-controls label { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; } +#app .run-time-controls :is(label, a) { color: var(--dim); font-size: 12px; } +#app .run-incidents { display: grid; gap: 8px; margin-bottom: 16px; } +#app .run-incident { display: flex; align-items: baseline; flex-wrap: wrap; gap: 4px 12px; font-size: 12px; } +#app .incident-time { color: var(--dim); font-family: var(--mono); } +#app .incident-error .incident-label { color: var(--status-error); } +#app .incident-warning .incident-label { color: var(--status-warning); } +#app .incident-context { color: var(--dim); overflow-wrap: anywhere; } +#app .run-incident a { color: var(--dim); white-space: nowrap; } +#app .run-metric-summary { display: flex; flex-wrap: wrap; gap: 8px 16px; margin-bottom: 16px; } +#app .run-activity { border-top: 0; padding-top: 0; } +#app .run-activity .activity-title { margin-top: 8px; margin-bottom: 4px; } +#app .run-activity .activity-intro { margin: 0 0 12px; font-size: 12px; } +#app .run-activity :is(.activity-header, .activity-row) { grid-template-columns: var(--activity-label-width) 88px minmax(0, 1fr) 48px; } +#app .run-activity .activity-gpu { color: var(--dim); font: 11px var(--mono); overflow-wrap: anywhere; } +#app .activity-gpu-toggle { padding: 0; border: 0; background: none; text-align: left; cursor: pointer; } +#app .activity-gpu-toggle:hover { color: var(--ink); } +#app .activity-chevron { display: inline-block; width: 12px; } +#app .activity-duty { padding: 10px 0 14px; border-top: 1px solid var(--line); } +#app :is(.activity-mode, .chart-mode) { display: inline-flex; gap: 4px; } +#app .activity-mode { margin: 0 0 12px; } +#app .chart-mode { margin-left: 12px; margin-right: auto; } +#app .activity-duty .chart { --plot-height: 100px; min-height: calc(var(--plot-height) + 82px); } +/* The plot lines up with the lane tracks: past the label and GPU columns, short of the duty column. */ +#app .activity-duty .chart-plot { grid-template-columns: calc(var(--activity-label-width) + 100px) minmax(0, 1fr); column-gap: 12px; margin-right: 60px; } +#app .run-activity .activity-row[data-continued] { min-height: 30px; border-top-color: transparent; } +#app .run-activity .activity-row[data-continued] .activity-track { height: 30px; } +#app .run-activity .activity-row[data-lead] { min-height: 30px; } +#app .run-activity .activity-row[data-lead] .activity-track { height: 30px; } +/* A group's label overhangs the empty label cells of its later rows, so every GPU row keeps one pitch. */ +#app .run-activity .activity-row[data-lead] .activity-label { align-self: start; align-items: flex-start; height: 30px; overflow: visible; } +#app .activity-source { margin-top: 12px; color: var(--dim); font-size: 11px; } +#app .run-metrics { margin-top: 8px; } +#app .time-control.run-time-controls { grid-template-columns: 32px minmax(0, 1fr) 32px auto auto; width: 520px; } +#app .log-toolbar { display: flex; flex-wrap: wrap; gap: 8px; margin-bottom: 12px; } +#app #run-panel > p + .log-toolbar { margin-top: 8px; } +#app .log-toolbar input { flex: 1; min-width: 100px; } +#app .log-toolbar select { max-width: min(100%, 340px); } +#app .log-toolbar .log-scope { display: flex; align-items: center; gap: 6px; color: var(--dim); font-size: 12px; white-space: nowrap; } +#app .log-toolbar .log-scope input { flex: none; min-width: 0; margin: 0; } +/* Fits "Jump to latest", so pausing by scrolling doesn't shift the toolbar. */ +#app .log-toolbar button { min-width: 8.75em; } +#app #log-status { display: grid; gap: 6px; color: var(--dim); font-size: 12px; } +#app #log-status:not(:empty) { margin-bottom: 12px; } +#app .source-error { margin: 8px 0; color: var(--status-warning); font-size: 12px; overflow-wrap: anywhere; } +#app .snapshot-error:not([hidden]) { display: flex; align-items: center; justify-content: space-between; gap: 8px 16px; margin: 16px var(--gutter) 0; padding: 12px; border: 1px solid color-mix(in srgb, var(--status-warning) 35%, var(--line)); border-radius: var(--radius); color: var(--status-warning); background: var(--paper); overflow-wrap: anywhere; } +#app .snapshot-error button { flex: none; } +#app pre, #app .workspace-logrow { white-space: pre-wrap; overflow-wrap: anywhere; font: 12px/1.65 var(--mono); } +/* At the bottom of the page the tabs, toolbar and lines fill the window; at the box's ends the wheel scrolls the page. */ +#app #log-lines { max-height: calc(100svh - 170px); overflow: auto; overflow-anchor: none; scrollbar-gutter: stable; } +#app .workspace-logrow { display: grid; grid-template-columns: 12ch var(--tag-width) minmax(0, 1fr); gap: 10px; padding: 1px 6px; border-left: 2px solid transparent; line-height: 1.5; } +#app .workspace-logrow[data-severity="error"] { border-left-color: var(--status-error); background: color-mix(in srgb, var(--status-error) 7%, transparent); } +#app .workspace-logrow[data-severity="warning"] { border-left-color: var(--status-warning); background: color-mix(in srgb, var(--status-warning) 7%, transparent); } +#app .log-time { color: var(--dim); white-space: nowrap; font-variant-numeric: tabular-nums; } +#app .log-pod { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--allocation-color, var(--dim)); } +#app .log-message { min-width: 0; margin: 0; font: inherit; } +#app .log-message mark { border-radius: 2px; background: color-mix(in srgb, var(--hue-3) 30%, transparent); color: inherit; } +#app .log-older { margin: 0 0 8px; } +#app .log-error { color: var(--status-error); } +#app .log-time-scope { display: flex; align-items: baseline; flex-wrap: wrap; gap: 8px 16px; margin-bottom: 12px; color: var(--dim); font-size: 12px; } +#app .log-time-scope a { color: var(--ink); } +#app .run-json-link { margin-top: 20px; color: var(--dim); font-size: 12px; } +/* Section titles and wide tables on the Run and Health pages. */ +#app .scheduler-title { margin: 24px 0 12px; font-size: 15px; font-weight: 500; line-height: 1.4; } +#app :is(.scheduler-table-wrap, .table-scroll) { overflow-x: auto; } +#app .scheduler-table { min-width: 650px; } +/* Health: only the evidence wraps, so pod names and links stay whole. */ +#app .scheduler-table :is(th, td):not(:nth-child(3)) { white-space: nowrap; } +/* Keep the identity readable before switching to compact rows. */ +@media (min-width: 761px) and (max-width: 1100px) { + #app :is(.job-list-head, .job-list-row).run-row { grid-template-columns: minmax(0, 1fr) 112px 56px 72px 140px; gap: 16px; } +} +@media (max-width: 760px) { + #app :is(.job-list-head, .job-list-row).run-row { grid-template-columns: minmax(0, 1fr) minmax(100px, auto); gap: 6px 12px; } + #app .job-list-head.run-row > :nth-child(n + 3) { display: none; } + #app .run-row > :nth-child(2) { justify-self: end; } + #app .run-row > :nth-child(n + 3) { color: var(--dim); font-size: 12px; } + #app .run-row > :nth-child(3) { grid-column: 1; text-align: left; } + #app .run-row > :nth-child(4) { grid-column: 2; text-align: right; } + #app .job-list-row.run-row > :nth-child(3)::after { content: " steps"; } + #app .job-list-row.run-row > :nth-child(4)::before { content: "Elapsed "; } + #app .job-list-row.run-row > .run-session { grid-column: 1 / -1; text-align: left; } +} +@media (max-width: 580px) { + #app .time-control { width: 100%; } + #app { --gutter: 14px; --gpu-index-width: 24px; --gpu-index-gap: 4px; } + #app .appbar { padding-block: 12px; gap: 8px 16px; } + #app .appbar > nav { order: 3; width: 100%; flex-wrap: nowrap; gap: 16px; overflow-x: auto; white-space: nowrap; font-size: 12px; } + #app #run-search { max-width: none; } + #app .pad { padding-block: 16px; } + #app .heading { font-size: 20px; } + #app :is(.node-lane, .node-time-header) { grid-template-columns: minmax(0, 1fr) 36px; gap: 6px 8px; } + #app .node-time-header > span:first-child { display: none; } + #app .node-lane-label { grid-column: 1 / -1; min-height: 20px; gap: 6px; font-size: 12px; } + #app .time-control { grid-template-columns: 32px minmax(0, 1fr) 32px; } + #app .time-control > [data-copy-view] { grid-column: 1 / -1; justify-self: end; } + #app .time-popover { right: 0; } + #app .claim-label { padding-inline: 3px; font-size: 10px; } + #app .node-axis span:not(:first-child):not(:last-child), #app .chart-x span:not(:first-child):not(:last-child) { display: none; } + #app .capacity-track::before { background: none; border-right: 1px solid var(--line); } + #app :is(.allocation-expansion, .cross-node-activity) { --activity-label-width: 40px; } + #app :is(.activity-header, .activity-row) { grid-template-columns: minmax(0, 1fr); } + #app .activity-header h3 { display: none; } + #app .activity-label { margin-top: 4px; } + #app .activity-track { height: 34px; } + #app .op-legend { margin-left: 0; } + #app .activity-axis span:not(:first-child):not(:last-child) { display: none; } + #app .run-activity :is(.activity-header, .activity-row) { grid-template-columns: 64px minmax(0, 1fr) 36px; } + #app .run-activity .activity-label { grid-column: 1 / -1; } + #app .run-activity .activity-axis { grid-column: 2; } + #app .run-activity .activity-row > span:empty { display: none; } + #app .chart-tip { max-width: 230px; white-space: normal; width: max-content; } + #app .run-toolbar { align-items: flex-start; } + #app .run-time-controls { width: 100%; justify-content: space-between; } + #app .log-toolbar > :is(input, select) { flex: 1 1 100%; width: 100%; min-width: 0; max-width: 100%; } + #app .workspace-logrow { grid-template-columns: auto minmax(0, 1fr); gap: 0 10px; padding-block: 4px; } + #app .log-message { grid-column: 1 / -1; } +} diff --git a/src/server/dashboard/ui/Activity.svelte b/src/server/dashboard/ui/Activity.svelte new file mode 100644 index 000000000..62f4a686c --- /dev/null +++ b/src/server/dashboard/ui/Activity.svelte @@ -0,0 +1,86 @@ + + tooltip = crosshair.at = null} onresize={() => tooltip = crosshair.at = null} /> + +
tooltip = crosshair.at = null} onpointerdown={() => tooltip = crosshair.at = null}> + {#if acrossNodes}

GPU activity

One row per accelerator. Colored bars are ops the process on that GPU ran; grey is GPU time held between ops. Duty is the share of time a kernel ran on the GPU, measured by DCGM.

+
{/if} + {#if !shown.length}

{empty}

+ {:else} +
+

{acrossNodes ? "Process" : "Run"}

{#if acrossNodes}

GPU

{/if}
{#each [0, 1, 2, 3, 4] as tick}{axisTime(range.start + (range.now - range.start) * tick / 4, range)}{/each}
{#if acrossNodes}Duty{/if}
+ {#each rows as { key, lane: p, device, first, lead } (key)} + {@const name = acrossNodes ? processLabel(p, inventory.cluster.nodes, { gpus: false }) : p.label} + {@const meta = acrossNodes ? p.node ? shortNodeName(p.node, inventory.cluster.nodes) : p.ended && p.pod ? p.pod : "Node unassigned" : p.role || "process"} + {@const link = href(p)} + {@const status = p.error ? "Ops unavailable" : !p.intervals.length ? "No recorded activity" : ""} +
+ {#if first} + {name}{meta}{p.ended ? " · Process ended" : p.phase && p.phase !== "ready" ? ` · ${p.phase.replace("_", " ")}` : ""}{!link ? acrossNodes ? p.node ? " · Node no longer reported" : "" : p.run_ids?.length ? " · Run record unavailable" : " · No linked run" : ""} + {:else}{/if} + {#if acrossNodes}{#if device && p.node}{:else}{device?.label ?? "GPUs not recorded"}{/if}{/if} + +
onpick(Math.round(timeAt(event, range))))}> + {#if mode === "lanes"} + {#if p.waits?.length} inspect(event, p.waits, `Waiting for the GPU · ${rowLabel(p)}`, "From asking the time slicer to the grant")} />{/if} + + select(event, p)} onkeydown={(event) => select(event, p)} onpointermove={(event) => inspect(event, p.intervals, rowLabel(p), p.exact ? "GPU turns" : "Recorded operations")} /> + {#each Object.entries(p.ops).sort(([a], [b]) => opRank(a) - opRank(b)) as [op, spans] (op)} + + select(event, p)} onkeydown={(event) => select(event, p)} onpointermove={(event) => inspect(event, spans, `${opLabel(op)} · ${rowLabel(p)}`, "Worker-recorded op")} /> + {/each} + {/if} + {#if status && first}{status}{/if} +
+ {#if acrossNodes}{dutyText(device)}{/if} +
+ {#if (open[key] || mode === "duty") && device && p.node}
{/if} + {/each} + {#if ops.length && mode === "lanes"}
{#each ops as op}{opLabel(op)}{/each}held, unlabelled{#if shown.some((p) => p.waits?.length)}waiting for the GPU{/if}
{/if} +
+ {/if} + {#if error && shown.length}

Activity unavailable: {error}

{/if} + {#if warnings.length}

{warnings.join(" · ")}

{/if} + {#if showSource && activities.length}

{activitySource(activities)}

{/if} + +
diff --git a/src/server/dashboard/ui/App.svelte b/src/server/dashboard/ui/App.svelte new file mode 100644 index 000000000..35f902942 --- /dev/null +++ b/src/server/dashboard/ui/App.svelte @@ -0,0 +1,54 @@ + + + + { if (!document.hidden) inventory.retry(); }} /> +
+
+ OpenRL + + {connection} +
+ +
+ {#if inventory.data} + {#if view.page === "run"} + {#key view.id}{/key} + {:else if view.page === "nodes"} + {:else}{/if} + {:else if !inventory.error}Loading cluster…{/if} +
+
diff --git a/src/server/dashboard/ui/Chart.svelte b/src/server/dashboard/ui/Chart.svelte new file mode 100644 index 000000000..62a8b61e6 --- /dev/null +++ b/src/server/dashboard/ui/Chart.svelte @@ -0,0 +1,128 @@ + + + selectedAt = null} /> +
+

{title}

{#if readingSeconds}{#if bands.length}{/if}{/if}{#if data.length && mode !== "running"}Latest {valueText(data.at(-1)[1], unit)}{/if}
+ {#if !data.length}

{empty}

+ {:else} + +
selectedAt = crosshair.at = null} onfocusout={() => selectedAt = null} onkeydown={keyboard}> +
{#each scale.ticks as tick}{chartNumber(tick)}{/each}
+ + {#each scale.ticks as tick}{/each} + {#each paths as path}{#if path.area}{/if}{/each} + {#each bands as band, i (band.key)}{#each band.spans as [a, b]}{/each} + {#each paths as path}{#if path.area}{/if}{/each}{/each} + {#each paths as path}{/each} + {#if !hover && crosshairX !== null}{/if} + +
{#each Array.from({ length: tickCount + 1 }, (_, i) => start + (end - start) * i / (tickCount || 1)) as at}{label(at)}{/each}
+ +
+
{#if mode === "running"}Something ran {valueText(100 * running.reduce((sum, [a, b]) => sum + b - a, 0) / (end - start || 1), "%")} of the range{:else}Average {valueText(values.reduce((sum, value) => sum + value, 0) / values.length, unit)}Peak {valueText(Math.max(...values), unit)}{#if readingSeconds}{mode === "rolling" ? `Mean of the last minute's ${readingSeconds} s readings` : `One reading per ${readingSeconds} s`}{/if}{/if}{#each bands as band (band.key)}{band.label}{/each}{#if note}{note}{/if}
+ {/if} +
diff --git a/src/server/dashboard/ui/CopyLink.svelte b/src/server/dashboard/ui/CopyLink.svelte new file mode 100644 index 000000000..9ad745479 --- /dev/null +++ b/src/server/dashboard/ui/CopyLink.svelte @@ -0,0 +1,12 @@ + + diff --git a/src/server/dashboard/ui/GpuDuty.svelte b/src/server/dashboard/ui/GpuDuty.svelte new file mode 100644 index 000000000..7d6bfef66 --- /dev/null +++ b/src/server/dashboard/ui/GpuDuty.svelte @@ -0,0 +1,14 @@ + + + diff --git a/src/server/dashboard/ui/Logs.svelte b/src/server/dashboard/ui/Logs.svelte new file mode 100644 index 000000000..88fccce1f --- /dev/null +++ b/src/server/dashboard/ui/Logs.svelte @@ -0,0 +1,127 @@ + + +{#if view.event !== null} +
{logRange.since.slice(0, 10)} · {logRange.since.slice(11, 19)}–{logRange.until.slice(11, 19)}All logs
+{/if} +
+ + + {#if run.shared_runtime}{/if} + +
+
{status} + {#if error}{error}{records.length ? " · Showing previously fetched records" : ""}{/if} +
+ +
{ const box = event.currentTarget; if (view.end === null && box.scrollHeight - box.clientHeight - box.scrollTop > 24) pause(); }}> + {#if cursor} + {:else if records.length === 2000}

2,000 records shown. Narrow the time range or search to inspect more.

{/if} + {#each shown as record (record.id)} + {@const at = timestamp(record.timestamp)} +
+ {tag(record.pod)}
{#each match ? record.message.split(match) : [record.message] as part, i}{#if i % 2}{part}{:else}{part}{/if}{/each}{#if record.message_truncated}{"\n[Message truncated by source]"}{/if}
+
+ {:else}{#if !pending && !error}

{data?.note || "No logs match this time range and filter"}

{/if}{/each} +
diff --git a/src/server/dashboard/ui/Metrics.svelte b/src/server/dashboard/ui/Metrics.svelte new file mode 100644 index 000000000..65a3b5b8f --- /dev/null +++ b/src/server/dashboard/ui/Metrics.svelte @@ -0,0 +1,73 @@ + + +{#snippet placeholders(count)}
{#each Array.from({ length: count }) as _, i (i)}{/each}
{/snippet} + +
+ {#if steps.error}

{steps.error}{rows.length ? " · Showing previously fetched steps" : ""}

{/if} +

Client metrics

+ {#if allClient.length}
+ +
+ {#each ["key", ...groups, "all"] as key (key)}{/each} +
+
{/if} + {#if clientCharts.length}
+ {#each clientCharts as [key, title, unit = ""] (key)} row.client?.[key])} start={stepRange.start} end={stepRange.end} empty="No values" />{/each} +
{:else if allClient.length}

No client metric matches

{:else if loading}{@render placeholders(FIRST.length)}{:else if steps.data}

0 metrics

{/if} +

Trainer metrics

+ {#if trainerCharts.length}
+ {#each trainerCharts as [op, name] (`${op} ${name}`)} row.ops[op]?.metrics[name]?.mean)} start={stepRange.start} end={stepRange.end} tone="accent" empty="No values" />{/each} +
{:else if loading}{@render placeholders(3)}{:else if steps.data}

0 metrics

{/if} +

Operations

+
Completed steps {run.steps ?? "—"}
+ {#if loading}{@render placeholders(3)}{/if} +
+ {#each opNames as op (op)} row.ops[op] && row.ops[op].seconds / row.ops[op].count)} start={stepRange.start} end={stepRange.end} min={0} tone="accent" empty="No values" />{/each} + {#if steps.data && !rows.length}

No operations recorded

{/if} +
+

Samplers

+ + {#if run.shared_runtime}

The pool's samplers serve every run on this base model; see their load on Nodes.

+ {:else}
+ {#each SAMPLER_CHARTS as [key, title, unit, scale = 1] (key)}{/each} +
{/if} +
diff --git a/src/server/dashboard/ui/Nodes.svelte b/src/server/dashboard/ui/Nodes.svelte new file mode 100644 index 000000000..4baf52c54 --- /dev/null +++ b/src/server/dashboard/ui/Nodes.svelte @@ -0,0 +1,143 @@ + + scrollSelection = true} onkeydown={escapeKey} /> +
range, getNow: () => observed, onGesture: (value) => navigating = !!value }} data-crosshair={cursor !== null ? "" : undefined} style:--crosshair={`${cursor ?? 0}%`}> +

Accelerator nodes

+
+ +
+
+ {#each errors as error}

{error}

{/each} + {#if history.error}{/if} + {#if !inventory.cluster.available && !errors.length}

Kubernetes unavailable

{/if} + {#if unavailable}
{view.node ? "The selected node is no longer reported." : "The selected placement is not in retained history."}
{/if} +
Node
{#each [0, 1, 2, 3] as step}{axisTime(range.start + (range.now - range.start) * step / 3, range)}{/each}
Duty
+ {#each loading ? [] : nodes as item (item.name)} + {@const devices = item.devices || []} + {@const height = devices.length === 1 ? 30 : 22} + {@const placements = all.filter((p) => p.node === item.name && p.start <= range.now && p.end >= range.start)} + {@const current = range.live ? inventory.placements.filter((p) => p.node === item.name) : placements.filter((p) => p.start <= range.now && p.end >= range.now)} + {@const unmapped = placements.filter((p) => !p.devices.length || p.devices.some((id) => !devices.some((d) => d.id === id)))} +
+
+
{#if devices.length}
{#each devices as device (device.id)}{deviceLabel(device.name)}{/each}
crosshair.at = event.buttons ? null : timeAt(event, range)} onpointerleave={() => crosshair.at = null}> + {#each capacityBars(devices, placements, range, view.placement) as bar (bar.key)} + + {/each} +
{/if} + {#each unmapped as p (p.id)}{/each} + {#if unmapped.length}Device mapping unavailable{:else if !devices.length && item.gpu_capacity}

GPUs without DRA devices

{/if}
+ {dutyCycle(item)} +
+ {#if item.name === view.node} + {@const label = `${acceleratorLabel(item)} · ${shortNodeName(item.name, inventory.cluster.nodes)}`} +

{label}

+ {#if devices.length > 1}
+ + + {#each devices as device (device.id)}{@const value = lastValue(deviceSeries(metrics.devices.filter((d) => d.id === device.id), "utilization"), range)}{/each} +
{/if} +
+ {#if item.gpu_capacity}
+ {#if queries.utilization.error}{/if} +
{activitySource(visible)}GPU memory {Number.isFinite(gpuMemory) ? `${(gpuMemory / 1024).toFixed(1)} GiB` : "—"}{#if gpuMemoryError} · {Number.isFinite(gpuMemory) ? "Stale" : "Unavailable"}: {gpuMemoryError}{/if}
{/if} +
+ {/if} +
+ {:else}{#if loading}

Loading activity…

{:else if !errors.length && inventory.cluster.available}

No accelerator nodes reported by Kubernetes

{/if}{/each} +
diff --git a/src/server/dashboard/ui/QueryChart.svelte b/src/server/dashboard/ui/QueryChart.svelte new file mode 100644 index 000000000..9ec4a09a7 --- /dev/null +++ b/src/server/dashboard/ui/QueryChart.svelte @@ -0,0 +1,14 @@ + + +
+ [at, value * scale])} start={range.start} end={range.now} min={0} {navigating} empty={result.error || "No samples in this time range"} note={result.error && series.length ? `${result.error} · Showing previously fetched samples` : ""} /> +
diff --git a/src/server/dashboard/ui/Run.svelte b/src/server/dashboard/ui/Run.svelte new file mode 100644 index 000000000..a8927d96e --- /dev/null +++ b/src/server/dashboard/ui/Run.svelte @@ -0,0 +1,94 @@ + + +

← {view.back ? "Nodes" : "Overview"}

+{#if !run} +

Run

+ {#if error}

{error}

{:else}

{view.id ? "Loading run…" : "Choose a run from Overview"}

{/if} + {#if detail.error}{/if} +{:else} +

{title}

+ {#if error}

{error}

{/if} + {#if description}

{description}

{/if} +
+ {#each (run.pods || []).filter((p) => p.problem) as pod}

{pod.name}: {pod.problem}

{/each} + {#if !finished && run.shared_runtime}

Shared runtime: its workers also serve other runs

{/if} +
+
range, getNow: () => observed, onGesture: (value) => navigating = !!value }}> +
+ +
Agent JSON ↗
+
+
+ {#if tab === "logs"} + {#if validEvent}{/if} + {:else if tab === "metrics"} + {:else} +
location.hash = link("logs", { event: at, end: range.now })} />
+ {/if} +
+
+{/if} diff --git a/src/server/dashboard/ui/Status.svelte b/src/server/dashboard/ui/Status.svelte new file mode 100644 index 000000000..96827b198 --- /dev/null +++ b/src/server/dashboard/ui/Status.svelte @@ -0,0 +1,7 @@ + +{label} diff --git a/src/server/dashboard/ui/TimeControl.svelte b/src/server/dashboard/ui/TimeControl.svelte new file mode 100644 index 000000000..a65921e1a --- /dev/null +++ b/src/server/dashboard/ui/TimeControl.svelte @@ -0,0 +1,33 @@ + + { if (picker?.open && !picker.contains(event.target)) picker.open = false; }} /> + +
{timeLabel} +
+ + {#if range.live}Following current time{:else}{/if} + W/S zoom · A/D pan +
+
+ diff --git a/src/server/dashboard/ui/Views.svelte b/src/server/dashboard/ui/Views.svelte new file mode 100644 index 000000000..8f61b0e2a --- /dev/null +++ b/src/server/dashboard/ui/Views.svelte @@ -0,0 +1,81 @@ + + +{#snippet healthStatus(label, tone)} + {label} +{/snippet} + +{#if view.page === "overview"} +

Overview

+
+ +
+ {#each FILTERS as [key, label, title]} + + {/each} +
+
+
+ {#if shown.length} +
+ JobStatusStepsElapsedSession +
+ {#each shown as run (run.run_id)} + {@const label = runDescription(run, null)} + {@const model = (run.model || "Run").split("/").at(-1)} + {@const session = sessionOf(run)} +
+ {model} · {run.run_id.slice(0, 8)}{#if KINDS[run.fine_tuning_type] || run.fine_tuning_type}{` · ${KINDS[run.fine_tuning_type] || run.fine_tuning_type}`}{/if}{#if label}{label}{/if} + {run.steps ?? "—"}{elapsedTime(run, inventory.observed_at)} + {session ?? "—"} +
+ {/each} + {/if} +
+ {#if !shown.length}
{inventory.runs.length ? "No runs match the search and status filter" : "No runs recorded"}
{/if} +{:else if view.page === "health"} +

Health

+ {#each errors as error}

{@render healthStatus("Source unavailable", "warning")}{error}

{/each} + {#if issues.length} +
+ {#each issues as issue}{/each} +
IssueResourceEvidence
{@render healthStatus(issue.issue, issue.tone)}{issue.resource}{issue.evidence}{#if issue.link}{issue.label} ↗{/if}
+ {:else if inventory.cluster.available === true && !errors.length}

{@render healthStatus("Healthy", "success")}No pod problems, node problems or source errors.

{/if} +

Diagnostic JSON ↗ · API reference ↗

+{/if} diff --git a/src/server/dashboard/ui/format.js b/src/server/dashboard/ui/format.js new file mode 100644 index 000000000..f5c525432 --- /dev/null +++ b/src/server/dashboard/ui/format.js @@ -0,0 +1,27 @@ +export const duration = (total) => total < 90 ? `${Math.round(total)}s` : total < 5400 ? `${Math.round(total / 60)}m` : total < 172800 ? `${(total / 3600).toFixed(1)}h` : `${(total / 86400).toFixed(1)}d`; +const seconds = (value) => value === null || value === undefined || value === "" ? NaN : typeof value === "number" ? value : Date.parse(value) / 1000; +// A run is finished once the gateway marks it so, or once nothing serves it any more; a client +// that never deletes its model leaves the status active. +export const isFinished = (run) => run?.lifecycle === "ended"; +// When a finished run stopped: when it was marked complete, else its last completed work. +export const finishedAt = (run) => [seconds(run.completed_at), run.last_completed_at].find(Number.isFinite) ?? NaN; +// One label: the gateway's status once it is final, else the lifecycle. +export const runLabel = (run) => run.client_status !== "active" ? run.client_status : { ended: "ended", waiting: "queued", placed: "running" }[run.lifecycle] || "active"; +export function elapsedTime(run, observedAt) { + const stop = isFinished(run) ? finishedAt(run) : NaN; + const start = seconds(run.created_at), end = Number.isFinite(stop) ? stop : seconds(observedAt); + return Number.isFinite(start) && Number.isFinite(end) ? duration(Math.max(0, end - start)) : "—"; +} +export function shortNodeName(name, nodes = []) { + const full = String(name || "—"), parts = full.split("-"); + const others = nodes.map((n) => typeof n === "string" ? n : n.name).filter((n) => n && n !== full); + for (let i = parts.length - 1; i >= 0; i--) { + const suffix = parts.slice(i).join("-"); + if (suffix && !others.some((other) => other === suffix || other.endsWith(`-${suffix}`))) return suffix; + } + return full; +} +export const KINDS = { lora: "LoRA", full: "FFT", fft: "FFT" }; +export const runDescription = (run, title = run.name) => [run.display_name, run.recipe_name].filter((v, i, values) => v && v !== title && values.indexOf(v) === i).join(" · "); +export const escapeRegExp = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +export const family = (id) => `hue-${Array.from(id || "").reduce((sum, c) => sum + c.charCodeAt(0), 0) % 8}`; diff --git a/src/server/dashboard/ui/index.js b/src/server/dashboard/ui/index.js new file mode 100644 index 000000000..2383efde6 --- /dev/null +++ b/src/server/dashboard/ui/index.js @@ -0,0 +1,3 @@ +import { mount } from "svelte"; +import App from "./App.svelte"; +mount(App, { target: document.getElementById("app") }); diff --git a/src/server/dashboard/ui/node-time.js b/src/server/dashboard/ui/node-time.js new file mode 100644 index 000000000..cdc264211 --- /dev/null +++ b/src/server/dashboard/ui/node-time.js @@ -0,0 +1,124 @@ +import { crosshair, MAX_WINDOW } from "./state.svelte.js"; + +export const WINDOWS = [[60, "1 minute"], [600, "10 minutes"], [1800, "30 minutes"], [3600, "1 hour"], [10800, "3 hours"], [21600, "6 hours"], [86400, "24 hours"], [259200, "3 days"], [MAX_WINDOW, "7 days"]]; +const clamp = (value, low, high) => Math.max(low, Math.min(high, value)); +const SURFACES = ".capacity-track, .node-axis, .activity-track, .activity-axis, #placement-detail .chart-plot svg, #placement-detail .chart-empty"; + +// Native gesture listeners are scoped to the mounted Nodes component. Svelte +// reacts to view changes; the query window stays frozen until a gesture ends. +export function nodeTime(element, { view, getRange, getNow, onGesture }) { + let gesture = null, drag = null, pinch = null, settle, ignoreClickUntil = 0; + const events = new AbortController(), root = element.closest("#app"); + const listen = (target, name, handler, options = {}) => target.addEventListener(name, handler, { ...options, signal: events.signal }); + const fraction = (x, box) => clamp((x - box.left) / (box.width || 1), 0, 1); + function begin(kind) { + if (gesture) return; + gesture = { kind, duration: view.duration, end: view.end }; + onGesture(getRange()); + root?.classList.add("node-navigating"); + } + function select(duration, end) { + duration = clamp(Math.round(duration), 60, MAX_WINDOW); + view.duration = duration; + view.end = clamp(end, getNow() - MAX_WINDOW + duration, getNow()); + } + function finish(cancel = false) { + clearTimeout(settle); + if (drag?.pointerId !== undefined && element.hasPointerCapture(drag.pointerId)) element.releasePointerCapture(drag.pointerId); + drag = pinch = null; + if (!gesture) return; + if (cancel) { view.duration = gesture.duration; view.end = gesture.end; } + if (gesture.kind === "drag") ignoreClickUntil = performance.now() + 400; + gesture = null; + onGesture(null); + root?.classList.remove("node-navigating"); + } + function moveDrag(x, y, touch = false) { + if (!drag) return false; + const dx = x - drag.x, dy = y - drag.y; + if (!gesture) { + if (touch && Math.abs(dy) > 6 && Math.abs(dy) > Math.abs(dx)) { drag = null; return false; } + if (Math.abs(dx) < 6) return false; + begin("drag"); + if (drag.pointerId !== undefined) element.setPointerCapture(drag.pointerId); + } + select(drag.range.now - drag.range.start, drag.range.now - dx / (drag.box.width || 1) * (drag.range.now - drag.range.start)); + return true; + } + listen(element, "wheel", (event) => { + const track = event.target.closest(SURFACES); + if (!track || !(event.ctrlKey || event.metaKey) || (gesture && gesture.kind !== "wheel")) return; + event.preventDefault(); + begin("wheel"); + const range = getRange(), at = fraction(event.clientX, track.getBoundingClientRect()); + const delta = event.deltaY * (event.deltaMode === 1 ? 16 : event.deltaMode === 2 ? track.clientHeight : 1); + const duration = clamp(Math.round((range.now - range.start) * Math.exp(clamp(delta * 0.002, -1, 1))), 60, MAX_WINDOW); + select(duration, range.start + at * (range.now - range.start) + (1 - at) * duration); + clearTimeout(settle); + settle = setTimeout(() => finish(), 200); + }, { passive: false }); + listen(element, "pointerdown", (event) => { + const track = event.target.closest(SURFACES); + if (!track || event.pointerType === "touch" || event.button !== 0 || event.target.closest("[data-placement], a")) return; + finish(); + drag = { x: event.clientX, y: event.clientY, box: track.getBoundingClientRect(), range: getRange(), pointerId: event.pointerId }; + }); + listen(window, "pointermove", (event) => { if (drag?.pointerId === event.pointerId && moveDrag(event.clientX, event.clientY)) event.preventDefault(); }); + listen(window, "pointerup", (event) => { if (drag?.pointerId === event.pointerId) finish(); }); + listen(window, "pointercancel", (event) => { if (drag?.pointerId === event.pointerId) finish(true); }); + listen(element, "touchstart", (event) => { + const track = event.target.closest(SURFACES); + if (!track) return; + if (event.touches.length === 2) { + event.preventDefault(); + if (gesture?.kind === "wheel") finish(); + begin("pinch"); + gesture.kind = "pinch"; + drag = null; + const [a, b] = event.touches; + pinch = { box: track.getBoundingClientRect(), range: getRange(), x: (a.clientX + b.clientX) / 2, distance: Math.hypot(a.clientX - b.clientX, a.clientY - b.clientY) || 1 }; + } else if (event.touches.length === 1 && !event.target.closest("[data-placement], a")) { + const touch = event.touches[0]; + drag = { x: touch.clientX, y: touch.clientY, box: track.getBoundingClientRect(), range: getRange() }; + } + }, { passive: false }); + listen(element, "touchmove", (event) => { + if (pinch && event.touches.length === 2) { + event.preventDefault(); + const [a, b] = event.touches, x = (a.clientX + b.clientX) / 2; + const duration = clamp(Math.round((pinch.range.now - pinch.range.start) * pinch.distance / (Math.hypot(a.clientX - b.clientX, a.clientY - b.clientY) || 1)), 60, MAX_WINDOW); + const anchor = pinch.range.start + fraction(pinch.x, pinch.box) * (pinch.range.now - pinch.range.start); + select(duration, anchor + (1 - fraction(x, pinch.box)) * duration); + } else if (drag && event.touches.length === 1 && moveDrag(event.touches[0].clientX, event.touches[0].clientY, true)) event.preventDefault(); + }, { passive: false }); + listen(element, "touchend", () => { if (drag || pinch) finish(); }); + listen(element, "touchcancel", () => { if (drag || pinch) finish(true); }); + listen(element, "click", (event) => { + if (event.detail && performance.now() < ignoreClickUntil) { ignoreClickUntil = 0; event.preventDefault(); event.stopImmediatePropagation(); } + }, { capture: true }); + listen(window, "keydown", (event) => { + if (event.key === "Escape" && (gesture || drag || pinch)) { event.preventDefault(); event.stopImmediatePropagation(); finish(true); } + }, { capture: true }); + // Perfetto keys: W/S zoom around the crosshair (else the middle), A/D pan by a fifth of the window. + listen(window, "keydown", (event) => { + const key = event.key.toLowerCase(); + if (!["w", "a", "s", "d"].includes(key) || event.ctrlKey || event.metaKey || event.altKey || event.target.closest("input, select, textarea") || (gesture && gesture.kind !== "keys")) return; + event.preventDefault(); + begin("keys"); + const range = getRange(), span = range.now - range.start; + if (key === "a" || key === "d") { + select(span, range.now + (key === "d" ? 0.2 : -0.2) * span); + // Keep the crosshair under the pointer, which did not move. + if (crosshair.at !== null) crosshair.at += view.end - range.now; + } else { + const duration = clamp(Math.round(span * (key === "w" ? 0.8 : 1.25)), 60, MAX_WINDOW); + const at = crosshair.at === null ? 0.5 : clamp((crosshair.at - range.start) / span, 0, 1); + select(duration, range.start + at * span + (1 - at) * duration); + } + clearTimeout(settle); + settle = setTimeout(() => finish(), 300); + }); + listen(window, "blur", () => finish(true)); + listen(window, "hashchange", () => finish()); + return { destroy() { finish(); events.abort(); } }; +} diff --git a/src/server/dashboard/ui/state.svelte.js b/src/server/dashboard/ui/state.svelte.js new file mode 100644 index 000000000..56c67e43e --- /dev/null +++ b/src/server/dashboard/ui/state.svelte.js @@ -0,0 +1,156 @@ +import { untrack } from "svelte"; + +const queryString = (params) => { + const query = new URLSearchParams(Object.entries(params).filter(([, v]) => v !== null && v !== undefined && v !== "")).toString(); + return query ? `?${query}` : ""; +}; +export const viewLink = (path, params = {}) => `#${path}${queryString(params)}`; +export const inspectUrl = (resource, params = {}) => `/api/v1/inspect/${resource}${queryString(params)}`; +export const nodeLink = (placement, range) => viewLink("nodes", { placement, duration: range.now - range.start, end: range.now }); +export const now = (inventory) => Date.parse(inventory?.observed_at) / 1000 || Date.now() / 1000; +const cache = new Map(); +// The longest window the pages show, matching the inspection API's seven-day limit. +export const MAX_WINDOW = 7 * 86400; +export const saved = $state({ overview: "#overview", nodes: "#nodes" }); +const number = (params, key, fallback) => params.get(key)?.trim() && Number.isFinite(Number(params.get(key))) ? Number(params.get(key)) : fallback; +export function parseView(hash = location.hash) { + const [path, search = ""] = hash.slice(1).split("?"); + const [page, id = "", tab = "activity"] = path.split("/").map((part) => { try { return decodeURIComponent(part); } catch { return part; } }); + const params = new URLSearchParams(search), duration = Math.max(60, Math.min(MAX_WINDOW, Math.round(number(params, "duration", 1800)))); + const end = number(params, "end", null), event = number(params, "event", null), back = params.get("back"); + return { page: ["nodes", "run", "health"].includes(page) ? page : "overview", id, tab, explicitWindow: params.has("duration") || params.has("end"), + duration, end: end !== null && end >= duration && end <= 8.64e12 ? end : null, + event: event !== null && (end === null || (event >= end - duration && event <= end)) ? event : null, + back: back?.startsWith("#nodes?") ? back : null, + node: params.get("node"), placement: params.get("placement"), gpu: params.get("gpu") || "all", + q: params.get("q") || "", pod: params.get("pod") || "", + status: ["active", "queued", "finished"].includes(params.get("status")) ? params.get("status") : "all" }; +} +export const view = $state(parseView()); +export const rangeOf = (inventory, selection = view) => { + const observed = now(inventory), duration = Math.max(60, Math.min(MAX_WINDOW, Math.round(selection.duration || 1800))); + const end = Math.max(observed - MAX_WINDOW + duration, Math.min(observed, selection.end ?? observed)); + return { start: end - duration, now: end, live: selection.end === null }; +}; +// Grafana's shared crosshair: hovering any chart or timeline marks the same instant on the others. +export const crosshair = $state({ at: null }); +export const timeAt = (event, range) => { + const box = event.currentTarget.getBoundingClientRect(); + return range.start + Math.max(0, Math.min(1, (event.clientX - box.left) / (box.width || 1))) * (range.now - range.start); +}; +export const crosshairOffset = (range) => crosshair.at === null || crosshair.at < range.start || crosshair.at > range.now ? null : 100 * (crosshair.at - range.start) / (range.now - range.start || 1); +export function currentView(freeze = false, tab = view.tab) { + const { page, id, duration, end, back, event, q, pod, node, placement, gpu, status } = view; + if (page === "overview") return viewLink(page, { q, status: status === "all" ? null : status }); + if (page === "run") return viewLink(`run/${encodeURIComponent(id)}/${tab || "activity"}`, { duration, end: freeze ? end ?? now(cache.get(inspectUrl("inventory"))?.data) : end, back, event, q, pod }); + if (page === "nodes") return viewLink(page, { duration, end: freeze ? end ?? now(cache.get(inspectUrl("inventory"))?.data) : end, node, placement, gpu: node && gpu !== "all" ? gpu : null }); + return viewLink(page + (id ? `/${encodeURIComponent(id)}` : "")); +} +export async function get(url, signal, previous) { + const timeout = AbortSignal.timeout(15000); + let response; + try { response = await fetch(url, { cache: "no-store", headers: previous?.etag ? { "If-None-Match": previous.etag } : undefined, signal: signal ? AbortSignal.any([signal, timeout]) : timeout }); } + catch (error) { + if (signal?.aborted) throw error; + throw new Error(timeout.aborted ? "Dashboard request timed out. Try again." : "Cannot reach the dashboard. Check the connection and try again."); + } + if (response.status === 304 && previous?.etag) return previous; + if (!response.ok) { + const data = await response.json().catch(() => null); + const detail = Array.isArray(data?.detail) ? data.detail.map((item) => `${(item.loc || []).filter((part) => part !== "body").join(".")}: ${item.msg}`).join("; ") : data?.detail; + const error = new Error([detail, data?.error].find((v) => typeof v === "string") || ({ 401: "Authentication required", 403: "Access denied", 404: "Requested data not found", 429: "Too many requests. Try again shortly." }[response.status] || `Dashboard request failed (${response.status})`)); + error.status = response.status; + throw error; + } + const data = await response.json().catch(() => { throw new Error("The dashboard returned an invalid response. Try again."); }); + return { data, etag: response.headers.get("ETag") }; +} + +// Reuse recent responses across navigation; changing the key or unmounting aborts old work. +// A live view keys on the user's selection, so its moving window is read at each poll instead of restarting it. +class QueryState { + // Responses are immutable snapshots. Only replacing the tree needs to be reactive. + data = $state.raw(null); + error = $state(null); + status = $state(null); + pending = $state(false); + // A brief failure keeps the last response and retries; this says it may be out of date. + stale = $state(false); + retry = $state(() => {}); +} +// remember: whether responses go in the shared cache. A live view's window moves every poll, so its +// responses are never revisited and would only push out the ones that are. +export function createQuery(url, { poll = 0, pollWhen = () => true, scope = () => true, key = url, merge = (next) => next, remember = () => true } = {}) { + let previousScope; + const result = new QueryState(); + const selectedKey = $derived.by(key), selectedScope = $derived.by(scope); + $effect(() => { + const requestedKey = selectedKey, requestedScope = selectedScope; + return untrack(() => { + result.retry = () => {}; + if (requestedScope !== previousScope) result.data = result.error = result.status = null; + previousScope = requestedScope; + const initialUrl = requestedKey && url(); + if (!initialUrl) return; + result.data = cache.get(initialUrl)?.data ?? result.data; + const controller = new AbortController(); + let failures = 0, timer; + const schedule = (delay, retry) => { + if (delay) timer = setTimeout(() => { + if (!document.hidden && (retry || pollWhen())) load(); + else schedule(delay, retry); + }, delay); + }; + const load = result.retry = async () => { + const requested = url(); + if (!requested || result.pending || controller.signal.aborted) return; + clearTimeout(timer); + result.pending = true; + let retry = false; + try { + const { data, etag } = await get(requested, controller.signal, cache.get(requested)); + if (controller.signal.aborted) return; + failures = 0; + Object.assign(result, { error: data.error || null, status: null, stale: false }); + if (data !== result.data && (!result.error || !result.data)) result.data = merge(data, result.data); + if (!result.error && remember()) { + cache.set(requested, { data: result.data, etag }); + if (cache.size > 24) cache.delete([...cache.keys()].find((key) => key !== inspectUrl("inventory"))); + } + } catch (error) { + if (controller.signal.aborted) return; + // A dropped tunnel or a busy server is usually brief: keep what is shown, retry on the one + // schedule, and report the failure only if it lasts. A rejected request is reported at once. + failures += 1; + retry = (!error.status || error.status >= 500 || error.status === 429) && failures < 4; + if (retry) result.stale = Boolean(result.data); + else Object.assign(result, { error: error.message, status: error.status || null, stale: false }); + } + finally { + if (!controller.signal.aborted) { + result.pending = false; + schedule(retry ? 2000 * 2 ** (failures - 1) : poll, retry); + } + } + }; + load(); + return () => { controller.abort(); clearTimeout(timer); result.pending = false; }; + }); + }); + return result; +} + +export async function copyLink(url, success = "Link copied") { + try { await navigator.clipboard.writeText(url); return success; } + catch { + const focused = document.activeElement, input = Object.assign(document.createElement("textarea"), { value: url }); + input.style.cssText = "position:fixed;opacity:0"; + document.body.append(input); + input.select(); + let copied = false; + try { copied = document.execCommand("copy"); } catch {} + input.remove(); + focused?.focus({ preventScroll: true }); + return copied ? success : "Copy failed"; + } +} diff --git a/src/server/dashboard/ui/visualization.js b/src/server/dashboard/ui/visualization.js new file mode 100644 index 000000000..84feaf4e6 --- /dev/null +++ b/src/server/dashboard/ui/visualization.js @@ -0,0 +1,72 @@ +// Presentation geometry only: identity and attribution come from the inspection API. +import { family } from "./format.js"; +export { shortNodeName } from "./format.js"; +const numbers = ["standard", "compact"].map((notation) => new Intl.NumberFormat(undefined, { maximumSignificantDigits: 4, notation })); +const percent = new Intl.NumberFormat(undefined, { maximumFractionDigits: 1 }); +export const chartNumber = (value) => numbers[Number(Math.abs(value) >= 10000)].format(value); +export const chartTime = (at, seconds = false) => new Date(at * 1000).toISOString().slice(11, seconds ? 19 : 16); +export const valueText = (value, unit) => unit === "%" ? `${percent.format(value)}%` : `${chartNumber(value)}${unit ? ` ${unit}` : ""}`; +export const axisTime = (at, range) => `${range.now - range.start >= 86400 ? `${new Date(at * 1000).toISOString().slice(5, 10)} ` : ""}${chartTime(at, range.now - range.start <= 120)}`; +export const placementColor = (p) => family(p.run_ids?.[0] || p.runtime_id); +// The exporter's collection interval in k8s/deploy; each engine-activity reading averages it. +export const DCGM_READING_SECONDS = 10; +export const dutyBands = (lanes, device = null) => lanes.filter((p) => p.intervals.length && (!device || p.devices.includes(device))).map((p) => ({ key: p.id, label: `${p.label} · ${p.role || "process"}`, hue: placementColor(p), role: p.role, spans: p.intervals })); +export const deviceLabel = (name) => /(\d+)$/.exec(String(name ?? ""))?.[1] ?? String(name ?? ""); +export const acceleratorLabel = (node) => node.accelerator ? node.accelerator.replace(/^nvidia[- ]/i, "").replace(/-\d+gb$/i, "").replace(/-/g, " ").toUpperCase() : node.gpu_capacity ? "GPU" : "CPU"; +export function clipPlacements(placements = [], range, error = null, status = null) { + const clip = (spans = []) => spans.map(([a, b]) => [Math.max(range.start, a), Math.min(range.now, b)]).filter(([a, b]) => b > a); + // A request failure belongs to every lane; a lane's own error is only about its ops. + return placements.map((p) => ({ ...p, intervals: clip(p.intervals), ops: Object.fromEntries(Object.entries(p.ops || {}).map(([name, spans]) => [name, clip(spans)])), error: status ? error : p.error, errorStatus: status || p.errorStatus })); +} +export function activitySource(placements) { + if (!placements.length) return ""; + const source = placements.every((p) => p.exact) ? "GPU turns" : placements.some((p) => p.exact) ? "GPU turns and recorded operations" : "Recorded operations; may include GPU wait time"; + return `${source}. Gaps may include unrecorded activity.`; +} +export function processLabel(p, nodes, { gpus: named = true } = {}) { + const node = nodes.find((n) => n.name === p.node), devices = p.devices.map((id) => node?.devices.find((d) => d.id === id)); + // A lane rebuilt from recorded ops has no known GPUs, which is not zero GPUs. + const gpus = devices.length && devices.every(Boolean) ? `GPU${devices.length === 1 ? "" : "s"} ${devices.map((d) => deviceLabel(d.name)).join(", ")}` : p.device_count ? `${p.device_count} GPU${p.device_count === 1 ? "" : "s"}` : null; + const role = p.role || "Process", accelerator = node && acceleratorLabel(node); + return [role[0].toUpperCase() + role.slice(1), accelerator === "GPU" ? null : accelerator, named && gpus].filter(Boolean).join(" · "); +} +export const OP_ORDER = ["forward_backward", "forward", "optim_step", "sample", "init", "wake_up", "sleep", "weight_sync"]; +export const opRank = (name) => OP_ORDER.includes(name) ? OP_ORDER.indexOf(name) : OP_ORDER.length; +export const opLabel = (name) => name.replace(/_/g, " "); +export function claimLabel(node, placements, live) { + if (live && !node.ready) return "Offline"; + if (!node.gpu_capacity) return "CPU"; + if (node.openrl_enabled === false) return "Not enabled"; + if (!placements.every((p) => p.devices.length === p.device_count && p.devices.every((id) => node.devices.some((d) => d.id === id)))) return "— claimed"; + return `${new Set(placements.flatMap((p) => p.devices)).size}/${node.gpu_capacity} claimed`; +} +export function capacityBars(devices, placements, range, selected) { + // Every held GPU draws the turns and ops its holders ran, never one block naming the run; the name is a click away. + const height = devices.length === 1 ? 30 : 22; + const span = (from, to) => `left:${100 * Math.max(0, Math.max(range.start, from) - range.start) / (range.now - range.start)}%;width:${100 * Math.max(0, Math.min(range.now, to) - Math.max(range.start, from)) / (range.now - range.start)}%`; + return devices.flatMap((device, i) => { + const rows = placements.filter((p) => p.devices.includes(device.id)).sort((a, b) => a.start - b.start || a.id.localeCompare(b.id)); + if (!rows.length) return []; + const from = Math.max(range.start, Math.min(...rows.map((p) => p.start))), to = Math.min(range.now, Math.max(...rows.map((p) => p.end))); + return [{ key: `gpu:${device.id}`, placement: rows.find((p) => p.id === selected) || rows.at(-1), selected: rows.some((p) => p.id === selected), title: `GPU activity: ${rows.map((p) => `${p.label} ${p.role || ""}`.trim()).join(", ")}`, + style: `${span(from, to)};top:${i * height + 2}px;height:${height - 4}px`, + holds: rows.flatMap((p) => p.intervals.map(([a, b]) => ({ placement: p, style: `left:${100 * (Math.max(a, from) - from) / (to - from || 1)}%;width:${100 * (Math.min(b, to) - Math.max(a, from)) / (to - from || 1)}%` }))) }]; + }); +} +// sortedPoints must be in time order; unsorted input gives wrong gaps without an error. +export function gapFor(sortedPoints, floor) { + const spacings = sortedPoints.slice(1).map(([at], i) => at - sortedPoints[i][0]).filter((d) => d > 0).sort((a, b) => a - b); + return Math.max(floor, 3 * (spacings[Math.floor(spacings.length / 2)] || 0)); +} +// Aggregate one physical generation per slot and timestamp; incomplete or ambiguous samples stay gaps. +export function deviceSeries(devices, field) { + const slots = new Set(devices.map((d) => d.id)).size, byTime = new Map(); + for (const device of devices) for (const [at, value] of device[field] || []) { + if (!byTime.has(at)) byTime.set(at, new Map()); + const values = byTime.get(at), samples = values.get(device.id) || []; + if (Number.isFinite(value)) samples.push(value); + values.set(device.id, samples); + } + return [...byTime].sort((a, b) => a[0] - b[0]).map(([at, values]) => [at, values.size === slots && [...values.values()].every((samples) => samples.length === 1) ? [...values.values()].reduce((sum, samples) => sum + samples[0], 0) / (field === "utilization" ? slots : 1) : null]); +} +export const lastValue = (points, range) => points.findLast(([at]) => at >= range.start && at <= range.now)?.[1]; diff --git a/src/server/model_metadata.py b/src/server/model_metadata.py index 0c4efe613..77584f1ed 100644 --- a/src/server/model_metadata.py +++ b/src/server/model_metadata.py @@ -15,6 +15,13 @@ SPARSE_DELTA_VERSION = 2 +def display_metadata(value: dict[str, Any]) -> dict[str, str]: + """Only bounded, explicitly supplied labels become run display metadata.""" + return { + key: value[key].strip()[:1024] for key in ("name", "run_name", "recipe_name", "git_rev") if isinstance(value.get(key), str) and value[key].strip() + } + + @dataclass class WeightSyncConfig: """How the trainer publishes weights each step: sparse deltas or full checkpoints. diff --git a/src/server/scheduler_worker_manager.py b/src/server/scheduler_worker_manager.py index 9da063f99..b6abbb1d7 100644 --- a/src/server/scheduler_worker_manager.py +++ b/src/server/scheduler_worker_manager.py @@ -28,6 +28,7 @@ from kubernetes import client, config from server.estimator import Footprint, footprint +from server.telemetry import SAMPLER_METRICS_PORT from server.worker_manager import ( FAILED, READY, @@ -47,6 +48,7 @@ GROUP = "openrl.io" VERSION = "v1alpha1" PLURAL = "workloads" +OPS_SETTINGS = ("OPEN_RL_OPS_TTL_DAYS", "OPEN_RL_OPS_SAMPLE_LIMIT") def workload_name(role: str, owner: str, is_lora: bool, index: int = 0) -> str: @@ -120,7 +122,8 @@ def pod_env(worker: Worker) -> list[dict[str, Any]]: } # MAX_JOBS caps FlashInfer's JIT build, which otherwise runs one ~3GB # compiler per core and blows through the pod's host memory limit. - for name in ("VLLM_GPU_MEMORY_UTILIZATION", "VLLM_MAX_MODEL_LEN", "VLLM_ENABLE_MULTIMODAL", "OPEN_RL_TRAIN_TOKEN_BUDGET", "MAX_JOBS"): + settings = ("VLLM_GPU_MEMORY_UTILIZATION", "VLLM_MAX_MODEL_LEN", "VLLM_ENABLE_MULTIMODAL", "OPEN_RL_TRAIN_TOKEN_BUDGET", "MAX_JOBS") + for name in (*settings, *OPS_SETTINGS): if os.getenv(name): values[name] = os.environ[name] # vLLM keys its torch.compile artifacts by model, GPU type and engine config. @@ -134,6 +137,9 @@ def pod_env(worker: Worker) -> list[dict[str, Any]]: values["OPEN_RL_TIME_SLICING"] = "off" env: list[dict[str, Any]] = [{"name": name, "value": value} for name, value in values.items()] env.append({"name": "OPEN_RL_ACCEL_TIMESLICER_HOST", "valueFrom": {"fieldRef": {"fieldPath": "status.hostIP"}}}) + # The pod identity a worker stamps on the ops it records, for the dashboard. + for name, field in (("POD_NAME", "metadata.name"), ("POD_UID", "metadata.uid"), ("NODE_NAME", "spec.nodeName")): + env.append({"name": name, "valueFrom": {"fieldRef": {"fieldPath": field}}}) return env @@ -185,6 +191,8 @@ def pod_template(worker: Worker) -> dict[str, Any]: if pull_policy := os.getenv("OPEN_RL_WORKER_IMAGE_PULL_POLICY"): template["spec"]["containers"][0]["imagePullPolicy"] = pull_policy + if worker.role == "sampler": + template["spec"]["containers"][0]["ports"] = [{"name": "http", "containerPort": SAMPLER_METRICS_PORT}] return template @@ -331,6 +339,22 @@ def load_workers(self) -> list[dict[str, Any]]: ) return workers + def workloads(self) -> list[dict[str, Any]]: + """Our Workload objects as the API returns them, read fresh, for the placement records.""" + selector = "app.kubernetes.io/managed-by=open-rl-api-server" + return self.custom_api.list_namespaced_custom_object(GROUP, VERSION, self.namespace, PLURAL, label_selector=selector)["items"] + + def pod_terminations(self) -> dict[str, list[dict[str, Any]]]: + """Each worker pod's container terminations, the current one and the one before a restart, for the placement records.""" + found: dict[str, list[dict[str, Any]]] = {} + for pod in client.CoreV1Api().list_namespaced_pod(self.namespace, label_selector="openrl.io/worker").items: + for cs in pod.status.container_statuses or []: + for ended in (cs.state and cs.state.terminated, cs.last_state and cs.last_state.terminated): + if ended and ended.finished_at: + entry = {"at": ended.finished_at.timestamp(), "container": cs.name, "reason": ended.reason, "exit_code": ended.exit_code} + found.setdefault(pod.metadata.name, []).append(entry | {"signal": ended.signal}) + return found + def worker_state(phase: str, reason: str, node: str | None, deleting: bool) -> tuple[str, str]: """Map a Workload's status onto the client vocabulary. The raw facts go diff --git a/src/server/session_registry.py b/src/server/session_registry.py index ac2c40167..1adf27d0e 100644 --- a/src/server/session_registry.py +++ b/src/server/session_registry.py @@ -8,36 +8,75 @@ Everything lives in the store, so an API server restart keeps it: - open_rl:session: present while the session is live. Each heartbeat - resets its expiry, so a silent session vanishes - on its own. Its value is the user_metadata the - client opened the session with, plus its openrl. - tags, the defaults for every model it creates. - open_rl:owner: the sessions using this owner's workers. - open_rl:owners every owner that has workers. + open_rl:session: present while the session is live. Each heartbeat + resets its expiry, so a silent session vanishes + on its own. Its value is the user_metadata the + client opened the session with, plus its openrl. + tags, the defaults for every model it creates. + open_rl:session_record: when the session started and was last heard + from, its redacted metadata and the models it + attached. It outlives the live key, for the + inspection API. It is telemetry, so it is + written in the background and a failed write + is dropped. + open_rl:owner: the sessions using this owner's workers. + open_rl:owners every owner that has workers. Nothing here is atomic. The API server holds a lock per owner around attach and around the in_use check and the teardown that follows it, so a session cannot attach to an owner between the check and the delete. """ +import asyncio import json +import time +from collections import defaultdict from typing import Any from server.store import StateStore +from server.telemetry import TTL_SECONDS, best_effort, key, log_event, redact, write class SessionRegistry: def __init__(self, state: StateStore, ttl_seconds: float = 120.0): self.state = state self.ttl_seconds = ttl_seconds + self.record_locks: defaultdict[str, asyncio.Lock] = defaultdict(asyncio.Lock) # the API server runs as one replica async def heartbeat(self, session_id: str) -> None: - key = f"open_rl:session:{session_id}" - await self.state.set_value(key, await self.state.get_value(key) or "{}", ttl_seconds=self.ttl_seconds) + await self.keep_alive(session_id) + self.note(session_id) + + async def keep_alive(self, session_id: str) -> None: + live_key = f"open_rl:session:{session_id}" + await self.state.set_value(live_key, await self.state.get_value(live_key) or "{}", ttl_seconds=self.ttl_seconds) async def update_metadata(self, session_id: str, user_metadata: dict[str, Any]) -> None: await self.state.set_value(f"open_rl:session:{session_id}", json.dumps(user_metadata), ttl_seconds=self.ttl_seconds) + self.note(session_id, metadata=user_metadata) + + @best_effort + def note(self, session_id: str, *, model_id: str | None = None, metadata: dict | None = None) -> None: + """A session's creation, heartbeat or model attach, kept on its record. Creation and attach also log a line; a + heartbeat, every ten seconds, only moves last_seen_at.""" + write(self.update_record(session_id, model_id, metadata)) + if metadata is not None: + log_event("session_created", {"session_id": session_id, "metadata": metadata}, session_id=session_id) + if model_id: + log_event("model_attached", {"session_id": session_id, "model_id": model_id}, run_id=model_id, session_id=session_id) + + async def update_record(self, session_id: str, model_id: str | None, metadata: dict | None) -> None: + record_key, now = key("session_record", session_id), time.time() + # Heartbeats and attaches rewrite the same record. One at a time, or a heartbeat drops a model attached meanwhile. + async with self.record_locks[session_id]: + saved = await self.state.get_value(record_key) + record = json.loads(saved) if saved else {"session_id": session_id, "created_at": now, "model_ids": []} + record["last_seen_at"] = now + if metadata is not None: + record["metadata"] = redact(metadata) + if model_id and model_id not in record["model_ids"]: + record["model_ids"].append(model_id) + await self.state.set_value(record_key, json.dumps(record), ttl_seconds=TTL_SECONDS) async def user_metadata(self, session_id: str | None) -> dict[str, Any]: raw = await self.state.get_value(f"open_rl:session:{session_id}") if session_id else None @@ -50,7 +89,7 @@ async def live(self, session_id: str) -> bool: async def attach(self, session_id: str, owner: str) -> None: """The session is using this owner's workers from now on.""" - await self.heartbeat(session_id) + await self.keep_alive(session_id) await self.state.add_to_set("open_rl:owners", owner) await self.state.add_to_set(f"open_rl:owner:{owner}", session_id) diff --git a/src/server/store.py b/src/server/store.py index 59c3d411b..a33f846e0 100644 --- a/src/server/store.py +++ b/src/server/store.py @@ -322,12 +322,64 @@ async def remove_from_set(self, key: str, member: str) -> None: async def set_members(self, key: str) -> set[str]: pass + @abstractmethod + def set_value_sync(self, key: str, value: str, ttl_seconds: float | None = None) -> None: + """set_value without the event loop, for a thread.""" + pass + + @abstractmethod + async def append_sample(self, key: str, sample: dict[str, Any], limit: int, ttl_seconds: float | None) -> None: + """Append to a list that keeps the newest `limit` samples and expires ttl_seconds after the last append, or never.""" + pass + + @abstractmethod + async def read_samples(self, key: str, last: int | None = None) -> list[dict[str, Any]]: + """The samples in order, or only the newest `last`.""" + pass + + @abstractmethod + async def replace_samples(self, key: str, samples: list[dict[str, Any]], ttl_seconds: float | None) -> None: + """Replace a list's samples in one step; it expires ttl_seconds later, or never.""" + pass + + @abstractmethod + async def expire(self, key: str, ttl_seconds: float) -> None: + """Expire a key ttl_seconds from now, whatever it holds.""" + pass + + @abstractmethod + async def increment(self, key: str, ttl_seconds: float | None) -> int: + """Atomically count up from zero; the counter expires ttl_seconds after the last increment, or never.""" + pass + + @abstractmethod + async def increment_field(self, key: str, field: str, ttl_seconds: float | None) -> int: + """increment for one of several counts kept under one key, which all expire together.""" + pass + + @abstractmethod + async def read_counts(self, key: str) -> dict[str, int]: + """The counts increment_field keeps under key.""" + pass + + @abstractmethod + async def scan_values(self, prefix: str) -> dict[str, str]: + """Every live value whose key starts with prefix.""" + pass + + @abstractmethod + async def list_length(self, key: str) -> int | None: + """Items in a list, such as a request queue; None where this store holds no queues.""" + pass + class InMemoryStateStore(StateStore): def __init__(self): self.kv_store: dict[str, str] = {} self.expiries: dict[str, float] = {} self.sets: dict[str, set[str]] = {} + self.samples: dict[str, list[str]] = {} + self.counts: dict[str, dict[str, int]] = {} async def set_value(self, key: str, value: str, ttl_seconds: float | None = None) -> None: self.kv_store[key] = value @@ -359,6 +411,46 @@ async def remove_from_set(self, key: str, member: str) -> None: async def set_members(self, key: str) -> set[str]: return set(self.sets.get(key, set())) + def set_value_sync(self, key: str, value: str, ttl_seconds: float | None = None) -> None: + self.kv_store[key] = value + self.expiries.pop(key, None) + if ttl_seconds is not None: + self.expiries[key] = time.monotonic() + ttl_seconds + + async def append_sample(self, key: str, sample: dict[str, Any], limit: int, ttl_seconds: float | None) -> None: + samples = self.samples.setdefault(key, []) + samples.append(json.dumps(sample, allow_nan=False)) + del samples[:-limit] + + async def read_samples(self, key: str, last: int | None = None) -> list[dict[str, Any]]: + return [json.loads(sample) for sample in self.samples.get(key, [])[-(last or 0) :]] + + async def replace_samples(self, key: str, samples: list[dict[str, Any]], ttl_seconds: float | None) -> None: + self.samples[key] = [json.dumps(sample, allow_nan=False) for sample in samples] + + async def expire(self, key: str, ttl_seconds: float) -> None: + if key in self.kv_store: + self.expiries[key] = time.monotonic() + ttl_seconds # lists, sets and counts here never expire + + async def increment(self, key: str, ttl_seconds: float | None) -> int: + value = int(self.get_value_sync(key) or 0) + 1 + await self.set_value(key, str(value), ttl_seconds) + return value + + async def increment_field(self, key: str, field: str, ttl_seconds: float | None) -> int: + counts = self.counts.setdefault(key, {}) + counts[field] = counts.get(field, 0) + 1 + return counts[field] + + async def read_counts(self, key: str) -> dict[str, int]: + return dict(self.counts.get(key, {})) + + async def scan_values(self, prefix: str) -> dict[str, str]: + return {key: value for key in list(self.kv_store) if key.startswith(prefix) and (value := self.get_value_sync(key)) is not None} + + async def list_length(self, key: str) -> int | None: + return None # the in-memory queues live in InMemoryStore + class RedisStateStore(StateStore): def __init__(self, client: redis.Redis, sync_client: sync_redis.Redis): @@ -387,6 +479,66 @@ async def remove_from_set(self, key: str, member: str) -> None: async def set_members(self, key: str) -> set[str]: return set(await self.redis.smembers(key)) + def set_value_sync(self, key: str, value: str, ttl_seconds: float | None = None) -> None: + self.sync_redis.set(key, value, px=None if ttl_seconds is None else int(ttl_seconds * 1000)) + + async def append_sample(self, key: str, sample: dict[str, Any], limit: int, ttl_seconds: float | None) -> None: + async with self.redis.pipeline(transaction=True) as pipeline: + pipeline.rpush(key, json.dumps(sample, allow_nan=False)) + pipeline.ltrim(key, -limit, -1) + if ttl_seconds: + pipeline.expire(key, int(ttl_seconds)) + else: + pipeline.persist(key) + await pipeline.execute() + + async def read_samples(self, key: str, last: int | None = None) -> list[dict[str, Any]]: + raw = await self.redis.lrange(key, -(last or 0), -1) + # Decoded off the event loop: a dashboard read can be thousands of samples. + return await asyncio.to_thread(lambda: [json.loads(sample) for sample in raw]) + + async def replace_samples(self, key: str, samples: list[dict[str, Any]], ttl_seconds: float | None) -> None: + async with self.redis.pipeline(transaction=True) as pipeline: + pipeline.delete(key) + if samples: + pipeline.rpush(key, *(json.dumps(sample, allow_nan=False) for sample in samples)) + if ttl_seconds: + pipeline.expire(key, int(ttl_seconds)) + await pipeline.execute() + + async def expire(self, key: str, ttl_seconds: float) -> None: + await self.redis.expire(key, int(ttl_seconds)) + + async def increment(self, key: str, ttl_seconds: float | None) -> int: + async with self.redis.pipeline(transaction=True) as pipeline: + pipeline.incr(key) + if ttl_seconds: + pipeline.expire(key, int(ttl_seconds)) + else: + pipeline.persist(key) + value, _ = await pipeline.execute() + return value + + async def increment_field(self, key: str, field: str, ttl_seconds: float | None) -> int: + async with self.redis.pipeline(transaction=True) as pipeline: + pipeline.hincrby(key, field, 1) + if ttl_seconds: + pipeline.expire(key, int(ttl_seconds)) + else: + pipeline.persist(key) + value, _ = await pipeline.execute() + return value + + async def read_counts(self, key: str) -> dict[str, int]: + return {field: int(value) for field, value in (await self.redis.hgetall(key)).items()} + + async def scan_values(self, prefix: str) -> dict[str, str]: + keys = [key async for key in self.redis.scan_iter(match=prefix + "*", count=1000)] + return {key: value for key, value in zip(keys, await self.redis.mget(keys)) if value is not None} if keys else {} + + async def list_length(self, key: str) -> int | None: + return await self.redis.llen(key) + @cache def redis_client(redis_url: str) -> redis.Redis: diff --git a/src/server/telemetry.py b/src/server/telemetry.py new file mode 100644 index 000000000..7bf3101ca --- /dev/null +++ b/src/server/telemetry.py @@ -0,0 +1,550 @@ +"""The records OpenRL keeps about its own runs, written where things happen, for the inspection API to read. + +A run's events go through record(), which keeps them in the store and also prints each as one JSON line labelled with +the run, for history. Sessions, placements and SDK events print the same lines through log_event(). + +Telemetry never changes what it observes. Every hook swallows its own errors, every store write runs in the background +and is dropped after WRITE_SECONDS, and no hook changes what an op returns or raises. +""" + +import asyncio +import functools +import json +import math +import os +import re +import socket +import threading +import time +import uuid +from collections import defaultdict +from collections.abc import Awaitable, Callable, Coroutine +from contextlib import asynccontextmanager, suppress +from dataclasses import dataclass +from datetime import datetime +from numbers import Real + +from server.store import get_state_store + +TTL_SECONDS = float(os.getenv("OPEN_RL_OPS_TTL_DAYS", "7")) * 86400 or None # 0 keeps records forever +SAMPLE_LIMIT = int(os.getenv("OPEN_RL_OPS_SAMPLE_LIMIT", "2000")) # command records kept per list +STEP_TIMES = 64 +SDK_ERRORS = 200 +MAX_TURNS = 200 +HEARTBEAT_SECONDS = 5 +STALE_SECONDS = 30 # a worker record not rewritten for this long belongs to a gone process +WRITE_SECONDS = 5 +FINISH_SECONDS = 5 # how long an exiting process waits for its last writes +OPEN_PLACEMENTS = "open_rl:placements:open" # models holding a placement not yet released +FAILED = "RequestFailedResponse" + +SECRET = re.compile(r"authorization|cookie|api[-_]?key|access[-_]?token|secret|password", re.I) +CONFIG_ENV = ("OPEN_RL_", "OPENRL_", "VLLM_", "NCCL_", "CUDA_", "PYTORCH_", "TORCH_") + + +def key(*parts: str) -> str: + return ":".join(("open_rl", *parts)) + + +def epoch(stamp: str | None) -> float | None: + return datetime.fromisoformat(stamp.replace("Z", "+00:00")).timestamp() if stamp else None + + +def rank_zero() -> bool: + # Every torchrun rank runs the same ops; rank 0 records them. + return os.getenv("RANK", "0") == "0" + + +def identity() -> dict[str, str]: + """The pod this process runs in, as the scheduler passes it.""" + return {field: value for field, env in (("pod", "POD_NAME"), ("pod_uid", "POD_UID"), ("node", "NODE_NAME")) if (value := os.getenv(env))} + + +def run_of(request: dict) -> str | None: + return request.get("logical_run_id") or request.get("model_id") + + +def operation_name(request: dict, role: str) -> str: + # A forward-only call travels as forward_backward, and its loss must not mix with training losses. + return "forward" if request.get("forward_only") else str(request.get("op") or ("sample" if role == "sampler" else "unknown")) + + +def redact(value): + # The SDK attaches full request and response headers to fatal errors, and the request carries the API key. + if isinstance(value, dict): + return {k: "[redacted]" if SECRET.search(str(k)) else redact(v) for k, v in value.items() if k not in ("request_headers", "response_headers")} + return [redact(v) for v in value] if isinstance(value, list) else value + + +writes: set[asyncio.Task] = set() # held so a write in flight is not garbage collected + + +def best_effort(hook): + @functools.wraps(hook) + def quiet(*args, **kwargs): + with suppress(Exception): + hook(*args, **kwargs) + + return quiet + + +def write(coroutine: Coroutine, timeout: float | None = None) -> None: + """Run a store write in the background. One that fails or takes longer than timeout (WRITE_SECONDS) is dropped.""" + + async def bounded(): + with suppress(Exception): + await asyncio.wait_for(coroutine, timeout or WRITE_SECONDS) + + try: + task = asyncio.get_running_loop().create_task(bounded()) + except RuntimeError: # no event loop to run it on + coroutine.close() + return + writes.add(task) + task.add_done_callback(writes.discard) + + +# Worker records + + +def gpu_uuids() -> list[str] | None: + """The GPUs this process can see, in CUDA order, as the UUIDs DCGM labels its series with. None when NVML cannot be + read. pynvml comes with vLLM, so a trainer may lack it, which says nothing about how many GPUs it has.""" + try: + import pynvml + + pynvml.nvmlInit() + uuids = [pynvml.nvmlDeviceGetUUID(pynvml.nvmlDeviceGetHandleByIndex(i)) for i in range(pynvml.nvmlDeviceGetCount())] + except Exception: + return None + visible = os.getenv("CUDA_VISIBLE_DEVICES") + return [gpu for i, gpu in enumerate(uuids) if visible is None or str(i) in visible.split(",") or gpu in visible] + + +def alive(record: dict, now: float) -> bool: + return record.get("phase") != "exited" and now - record.get("heartbeat_at", 0) < STALE_SECONDS + + +@dataclass +class Outstanding: + """A request this worker popped and has not answered.""" + + op: str + run_id: str | None + since: float + observed: bool = False # observe_operation records the op, so answered need not + + +class Worker: + """This process's worker record. A thread rewrites it, so it stays fresh while loading a model holds the event loop.""" + + def __init__(self, role: str, runtime_id: str | None) -> None: + now = time.time() + self.record = { + # One record per process, not per pod, so a container restarted in place does not overwrite its predecessor's. + "id": uuid.uuid4().hex[:16], + "node": socket.gethostname(), + **identity(), + "role": role, + "runtime_id": runtime_id, + "gpus": gpu_uuids(), + "started_at": now, + "phase": "starting", + "phase_since": now, + "turns": [], + "waits": [], + # The settings the scheduler passed as env, which is what this process runs with. + "config": {"env": {k: v for k, v in sorted(os.environ.items()) if k.startswith(CONFIG_ENV) and not SECRET.search(k)}}, + } + self.outstanding: dict[str, Outstanding] = {} + self.stopped = threading.Event() + self.heartbeat = threading.Thread(target=self.beat, name="worker-heartbeat", daemon=True) + + def snapshot(self) -> str: + """The record as JSON, with outstanding requests counted by op and run, each group showing its oldest.""" + groups: dict[tuple, dict] = {} + for entry in list(self.outstanding.values()): # a copy, since the event loop changes it while the heartbeat reads + group = groups.setdefault((entry.op, entry.run_id), {"op": entry.op, "run_id": entry.run_id, "since": entry.since, "count": 0}) + group["count"] += 1 + return json.dumps({**self.record, "heartbeat_at": time.time(), "outstanding": list(groups.values())}) + + @best_effort + def save(self) -> None: + write(get_state_store().set_value(key("worker", self.record["id"]), self.snapshot(), ttl_seconds=TTL_SECONDS)) + + def beat(self) -> None: + while not self.stopped.wait(HEARTBEAT_SECONDS): + with suppress(Exception): + get_state_store().set_value_sync(key("worker", self.record["id"]), self.snapshot(), ttl_seconds=TTL_SECONDS) + + +current: Worker | None = None # this process's record while it serves + +# The sampler pod's port named http, which the sampler PodMonitoring scrapes. +SAMPLER_METRICS_PORT = 9095 + + +@best_effort +def serve_metrics() -> None: + """Serve this process's Prometheus registry, where an in-process vLLM engine records its own metrics. Always in the + 0.0.4 text format, which keeps vLLM's vllm: names; offered OpenMetrics, the scraper asks for them escaped to vllm_.""" + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + from prometheus_client.exposition import CONTENT_TYPE_PLAIN_0_0_4, generate_latest # in the sampler image through vLLM + + class Metrics(BaseHTTPRequestHandler): + def do_GET(self) -> None: + body = generate_latest() + self.send_response(200) + self.send_header("Content-Type", CONTENT_TYPE_PLAIN_0_0_4) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args) -> None: # one line per scrape would flood the sampler's log + pass + + server = ThreadingHTTPServer(("", SAMPLER_METRICS_PORT), Metrics) + threading.Thread(target=server.serve_forever, name="sampler-metrics", daemon=True).start() + + +@asynccontextmanager +async def serving(role: str, runtime_id: str | None): + """Keep this process's worker record while the body runs, and mark it exited after.""" + global current + if rank_zero(): + with suppress(Exception): + current = Worker(role, runtime_id) + current.heartbeat.start() + current.save() + try: + yield current + finally: + await finish() + + +async def finish() -> None: + """Mark this process exited and land the writes in flight, within FINISH_SECONDS. Also called before an os._exit.""" + global current + worker, current = current, None + with suppress(Exception): + async with asyncio.timeout(FINISH_SECONDS): + if worker is not None: + worker.stopped.set() # stop the heartbeat first, so no late beat overwrites the exit + await asyncio.to_thread(worker.heartbeat.join, WRITE_SECONDS) + worker.record.update(phase="exited", phase_since=time.time()) + worker.save() + await asyncio.gather(*writes, return_exceptions=True) + + +@best_effort +def set_phase(phase: str) -> None: + if current is not None and current.record["phase"] != phase: + current.record.update(phase=phase, phase_since=time.time()) + + +@asynccontextmanager +async def gpu_turn(slot): + """Take the time slicer's slot for the body and show it on the worker record: the wait from asking to the grant, then + the turn, which is the hold.""" + asked = time.time() + async with slot: + started = time.time() + if current is not None: + current.record["waits"] = [*current.record["waits"][1 - MAX_TURNS :], [asked, started]] + set_phase("on_gpu") + try: + yield + finally: + if current is not None: + current.record["turns"] = [*current.record["turns"][1 - MAX_TURNS :], [started, time.time()]] + set_phase("ready") + + +@best_effort +def popped(requests: list[dict]) -> None: + """Requests this worker took from its queue, each outstanding until answered. A shared trainer's delete_model is queued + without being counted as enqueued, so it is not counted here either.""" + if current is None: + return + for request in requests: + if request.get("request_id") and request.get("op") != "delete_model": + run_id, op = run_of(request), operation_name(request, current.record["role"]) + current.outstanding[request["request_id"]] = Outstanding(op, run_id, time.time()) + record(run_id, "popped", request_id=request["request_id"], op=op) + + +@best_effort +def answered(request_id: str, result: dict | None = None) -> None: + """The request's answer is written. A failure answered without its op running, such as a failed wake or parse, gets its + command record here, so every failed answer has one.""" + if current is None or (entry := current.outstanding.pop(request_id, None)) is None: + return + record(entry.run_id, "answered", request_id=request_id, op=entry.op) + if not entry.observed and (result or {}).get("type") == FAILED: + request = {"op": entry.op, "request_id": request_id, "logical_run_id": entry.run_id} + op_finished(request, current.record["role"], current.record["runtime_id"], entry.since, FAILED) + + +# Command records + + +@dataclass +class RunSteps: + """A run's optimizer steps as this process counts them. start is the run's count when this process first ran one of + its ops, read in the background, and done_here the steps finished here since.""" + + start: asyncio.Task + done_here: int = 0 + + +run_steps: dict[str, RunSteps] = {} + + +async def read_step_count(run_id: str) -> int | None: + with suppress(Exception): + return int(await asyncio.wait_for(get_state_store().get_value(key("steps", run_id)), WRITE_SECONDS) or 0) + return None + + +async def observe_operation(request: dict, role: str, runtime_id: str | None, call: Callable[[], Awaitable[dict]]) -> dict: + """Run one op and record how it went. Returns or raises exactly as the op does.""" + if not rank_zero(): + return await call() + started_at = time.time() + with suppress(Exception): + if current is not None and (entry := current.outstanding.get(request.get("request_id"))): + entry.observed = True + # Read the run's step count before this op can change it. + if (run_id := run_of(request)) and run_id not in run_steps and operation_name(request, role) != "sample": + run_steps[run_id] = RunSteps(asyncio.ensure_future(read_step_count(run_id))) + try: + result = await call() + except Exception as error: + op_finished(request, role, runtime_id, started_at, type(error).__name__) + raise + op_finished(request, role, runtime_id, started_at, result=result) + return result + + +def numbers(metrics: dict | None) -> dict[str, float]: + """The metric values that are numbers, as floats. A bool is not one.""" + return {k: float(v) for k, v in (metrics or {}).items() if isinstance(v, Real) and not isinstance(v, bool)} + + +def finite(metrics: dict | None) -> dict[str, float]: + return {k: v for k, v in numbers(metrics).items() if math.isfinite(v)} + + +def command_record(request: dict, role: str, runtime_id: str | None, started_at: float, error_type: str | None, result) -> dict: + """How one op went. It failed with error_type, or as its result says. No payload, output or error message is kept.""" + result = result if isinstance(result, dict) else {} + if error_type is None and result.get("type") == FAILED: + error_type = FAILED + now, values = time.time(), numbers(result.get("metrics")) + record = { + "at": now, + "started_at": started_at, + "operation": operation_name(request, role), + "request_id": request.get("request_id"), + "run_id": run_of(request), + "runtime_id": runtime_id, + "role": role, + "status": "failed" if error_type else "succeeded", + "error_type": error_type, + "elapsed_seconds": now - started_at, + **identity(), + "metrics": finite(values), + } + # JSON has no NaN or infinity, so name the metrics that were, rather than leave them missing. + if nonfinite := sorted(k for k, v in values.items() if not math.isfinite(v)): + record["nonfinite"] = nonfinite + return record + + +@best_effort +def op_finished(request: dict, role: str, runtime_id: str | None, started_at: float, error_type: str | None = None, result=None) -> None: + """Record how one op went. A sample a vLLM sampler answered is only counted, with the time of the last, since vLLM's own + metrics time what it answers.""" + op = command_record(request, role, runtime_id, started_at, error_type, result) + if op["operation"] == "sample" and role == "sampler" and op["status"] == "succeeded": + count_sample(op["run_id"], op["at"]) + else: + record(op["run_id"], "op", **op) + + +@best_effort +def sampled_group(requests: list[dict], started_at: float) -> None: + """One record per run for the samples a vLLM sampler served together, so its lane shows when it sampled without a + record per sample. op_finished counted each sample, and a failed one keeps its own record too.""" + store, groups = get_state_store(), defaultdict(list) + for request in requests: + groups[run_of(request)].append(request) + for run_id, group in groups.items(): + if run_id: + op = command_record(group[0], "sampler", group[0].get("model_id"), started_at, None, None) | {"request_id": None, "count": len(group)} + write(store.append_sample(key("operations", run_id, "samples"), op, SAMPLE_LIMIT, TTL_SECONDS)) + log_event("op", op, run_id=run_id) + + +def count_sample(run_id: str | None, at: float) -> None: + if run_id: + store = get_state_store() + write(store.increment(key("sampled", run_id), TTL_SECONDS)) + write(store.set_value(key("sampled_at", run_id), str(at), ttl_seconds=TTL_SECONDS)) + + +def keep_op(run_id: str, op: dict) -> None: + """Keep one op's command record, and for a trainer op the run's failure and step counts.""" + store = get_state_store() + if op["operation"] == "sample": + count_sample(run_id, op["at"]) + # Samples keep their own list, so a run's sampling never evicts its training ops. + write(store.append_sample(key("operations", run_id, "samples"), op, SAMPLE_LIMIT, TTL_SECONDS)) + return + steps = run_steps.get(run_id) + # done_here is taken before this op's own step is counted below. The write may wait WRITE_SECONDS on the step read. + write(append_trainer_op(op, steps, steps.done_here if steps else 0), 2 * WRITE_SECONDS) + # The few records a run's one-line summary reads instead of its command history. + if op["status"] == "failed": + count_request(run_id, "failed") + elif op["operation"] == "optim_step": + if steps is not None: + steps.done_here += 1 + write(store.increment(key("steps", run_id), TTL_SECONDS)) + write(store.append_sample(key("step_times", run_id), op["at"], STEP_TIMES, TTL_SECONDS)) + + +async def append_trainer_op(record: dict, steps: RunSteps | None, done_here: int) -> None: + """Keep a trainer op, stamped with the step it belongs to once the run's count is read. An unread count leaves it off.""" + if steps is not None and (start := await steps.start) is not None: + record["step"] = start + done_here + await get_state_store().append_sample(key("operations", record["run_id"]), record, SAMPLE_LIMIT, TTL_SECONDS) + + +def count_request(run_id: str, event: str) -> None: + """Count one more of the run's requests reaching event, which is enqueued, popped, answered or failed. A request + enqueued but never popped shows up as a gap between the counts.""" + write(get_state_store().increment_field(key("ledger", run_id), event, TTL_SECONDS)) + + +# Run events + + +@best_effort +def record(run_id: str | None, kind: str, /, **fields) -> None: + """Keep one event about a run and log it. kind is enqueued, popped or answered, which count the run's requests, or op, + whose fields are a command record from command_record. The store write runs in the background.""" + if not run_id: + return + with suppress(Exception): + if kind == "op": + keep_op(run_id, fields) + else: + count_request(run_id, kind) + log_event(kind, fields, run_id=run_id) + + +@best_effort +def log_event(kind: str, fields: dict, *, run_id: str | None = None, session_id: str | None = None, severity: str | None = None) -> None: + """Print one event as a JSON line, labelled with its run or session, for the year of history Cloud Logging keeps. The + inspection API never reads these lines. A failed op logs as an error.""" + labels = {"openrl_source": "telemetry"} | ({"openrl_run_id": run_id} if run_id else {}) | ({"openrl_session_id": session_id} if session_id else {}) + line = {"severity": severity or ("ERROR" if fields.get("status") == "failed" else "INFO"), "message": f"telemetry {kind}"} + print(json.dumps(line | redact(fields) | {"logging.googleapis.com/labels": labels}, default=str), flush=True) + + +# Placement records + +placements_lock = asyncio.Lock() # sweeps run in the background and may overlap + + +def placement(workload: dict, terminations: dict[str, list[dict]]) -> dict: + """A Workload's placement, with the container terminations its pod has shown since, so they outlive the pod.""" + spec, status = workload["spec"], workload["status"] + placed_at = epoch(status["placedAt"]) + return { + "workload": workload["metadata"]["name"], + "role": spec["role"], + "model_id": spec["modelID"], + "owner": spec.get("ownerID"), + "pod": status.get("podName"), + "node": status.get("nodeName"), + "gpu_uuids": status.get("gpuUUIDs") or [], + "placed_at": placed_at, + "released_at": epoch(status.get("releasedAt")), + "terminations": [t for t in terminations.get(status.get("podName"), []) if t["at"] >= placed_at], + } + + +def merge_placements(records: list[dict], live: dict[str, dict], now: float) -> list[dict]: + """A model's placement records after a sweep. A record whose Workload still holds that placement takes its current + state, an open one whose Workload is gone or placed again is released now, and a new placement is added.""" + fresh, merged = dict(live), [] + for old in records: + if (new := fresh.get(old["workload"])) and new["placed_at"] == old["placed_at"]: + # A restarted container's earlier termination drops out of its pod status; the record keeps it. + kept = old.get("terminations", []) + merged.append(fresh.pop(old["workload"]) | {"terminations": kept + [t for t in new["terminations"] if t not in kept]}) + elif old["released_at"] is None: + merged.append({**old, "released_at": now}) + else: + merged.append(old) + return merged + list(fresh.values()) + + +async def record_placements(store, workloads: list[dict], terminations: dict[str, list[dict]]) -> None: + """Keep one record per Workload placement, which outlives the Workload, and the set of models still holding one.""" + now = time.time() + live: defaultdict[str, dict[str, dict]] = defaultdict(dict) # model -> Workload name -> placement + for workload in workloads: + if (workload.get("status") or {}).get("placedAt"): # the scheduler stamps it on bind + p = placement(workload, terminations) + live[p["model_id"]][p["workload"]] = p + async with placements_lock: + opened = await store.set_members(OPEN_PLACEMENTS) + for model_id in opened | live.keys(): + records_key = key("placements", model_id) + records = await store.read_samples(records_key) + merged = merge_placements(records, live[model_id], now) + held = any(r["released_at"] is None for r in merged) + if merged != records: + await store.replace_samples(records_key, merged, TTL_SECONDS) + for changed in (p for p in merged if p not in records): # placed, moved or released + log_event("placement", changed, run_id=model_id) + elif held and TTL_SECONDS: # a placement held longer than the TTL must outlive it + await store.expire(records_key, TTL_SECONDS) + if held and model_id not in opened: + await store.add_to_set(OPEN_PLACEMENTS, model_id) + elif not held and model_id in opened: + await store.remove_from_set(OPEN_PLACEMENTS, model_id) + + +# SDK events + + +@best_effort +def ingest_sdk_events(store, body: dict) -> None: + """Log the tinker SDK's own telemetry, one redacted JSON line per event, and keep a session's error events.""" + context = {k: body.get(k) for k in ("session_id", "process_uuid", "sdk_version", "platform")} + session_id, events = body.get("session_id"), body.get("events") + if not isinstance(events, list): + return + session_id = session_id if isinstance(session_id, str) and session_id else None + for event in [redact(e) for e in events if isinstance(e, dict)]: + severity = str(event.get("severity") or "INFO").upper() + log_event("sdk_event", {**context, "event": event}, session_id=session_id, severity=severity) + if severity in ("ERROR", "CRITICAL") and session_id: + write(store.append_sample(key("sdk_errors", session_id), sdk_error(event, severity), SDK_ERRORS, TTL_SECONDS)) + + +def sdk_error(event: dict, severity: str) -> dict: + data = event.get("event_data") if isinstance(event.get("event_data"), dict) else {} + name = str(event.get("error_type") or event.get("event_name") or event.get("event") or "event").rsplit(".", 1)[-1] + status = f"HTTP {data['status_code']}: " if data.get("status_code") else "" + message = status + str(event.get("error_message") or data.get("detail") or data.get("exception") or data.get("message") or "") + try: + at = datetime.fromisoformat(str(event.get("timestamp"))).timestamp() + except ValueError: + at = time.time() + return {"at": at, "severity": severity, "reason": f"SDK {name}", "message": message[:2000]} diff --git a/src/server/training_requests_processor.py b/src/server/training_requests_processor.py index 2ff259b0d..189aa147f 100644 --- a/src/server/training_requests_processor.py +++ b/src/server/training_requests_processor.py @@ -21,6 +21,7 @@ from accel_timeslicer.workload import TRAINER_CLAIM, local_workload_name from server.model_metadata import get_model_metadata from server.store import RequestStore, StateStore, get_state_store, get_store +from server.telemetry import answered, finish, gpu_turn, observe_operation, popped, serving, set_phase from training import commands from training.commands import parse_command from training.distributed import broadcast_object, is_distributed, is_primary, local_rank @@ -123,6 +124,7 @@ async def run(self) -> None: if self.time_slicer is not None: await self.time_slicer.register(self.workload) self.snapshot_registered = True + set_phase("ready") while True: try: await self.announce_readiness() @@ -180,6 +182,7 @@ async def exit_gracefully(self, unregister: bool = True) -> None: await self.time_slicer.close() except Exception: pass + await finish() os._exit(0) async def next_batch(self) -> list[dict[str, Any]]: @@ -202,6 +205,7 @@ async def publish_result(self, request_id: str | None, result: dict[str, Any]) - # Only rank 0 writes to the store; the other ranks computed the same result. if request_id is not None and is_primary(): await self.store.set_result(request_id, result) + answered(request_id, result) async def run_once(self) -> None: batch = await self.next_batch() @@ -211,6 +215,7 @@ async def run_once(self) -> None: shutdown = any(req.get("op") == "shutdown_workers" for req in batch) work = [req for req in batch if req.get("op") != "shutdown_workers"] + popped(work) model_id = batch[0].get("model_id", "default") results: list[tuple[str | None, dict[str, Any]]] = [] @@ -265,7 +270,7 @@ async def handle_batch(self, requests: list[dict[str, Any]], results: list[tuple save_reqs = [r for r in requests if r.get("op") in save_ops] if gpu_reqs: - async with self.time_slicer.acquire(self.workload): + async with gpu_turn(self.time_slicer.acquire(self.workload)): await asyncio.to_thread(self.worker.wake_up) try: for request in gpu_reqs: @@ -279,7 +284,7 @@ async def handle_batch(self, requests: list[dict[str, Any]], results: list[tuple for request in save_reqs: results.append(await self.handle_request(request)) else: - async with self.time_slicer.acquire(self.workload): + async with gpu_turn(self.time_slicer.acquire(self.workload)): for request in save_reqs: results.append(await self.handle_request(request)) @@ -298,7 +303,7 @@ async def handle_request(self, raw_request: dict[str, Any]) -> tuple[str | None, ctx = propagate.extract(command.trace_context) if command.trace_context else None token = otel_context.attach(ctx) if ctx else None - result = await self.dispatch_operation(command) + result = await observe_operation(raw_request, "trainer", self.model_id, lambda: self.dispatch_operation(command)) return request_id, result except Exception as exc: traceback.print_exc() @@ -464,6 +469,11 @@ def run_probe_server(): ) +async def serve_trainer(args: argparse.Namespace) -> None: + async with serving("trainer", os.getenv("OPEN_RL_MODEL_ID") or args.model_id): + await main_async(args) + + def start_request_processing_loop() -> None: parser = argparse.ArgumentParser() parser.add_argument("--model-id", help="Model id whose per-model request queue this dedicated trainer worker drains.") @@ -477,7 +487,7 @@ def start_request_processing_loop() -> None: cuda_devs = os.getenv("CUDA_VISIBLE_DEVICES", "ALL") print(f"-> Hardware : CUDA_VISIBLE_DEVICES={cuda_devs}") - asyncio.run(main_async(args)) + asyncio.run(serve_trainer(args)) if __name__ == "__main__": diff --git a/src/server/vllm_sampler.py b/src/server/vllm_sampler.py index 19fe43c9f..5ca8dbc6a 100644 --- a/src/server/vllm_sampler.py +++ b/src/server/vllm_sampler.py @@ -26,6 +26,7 @@ from accel_timeslicer.time_slicer import TimeSlicerClient, time_slicer_client_from_env, time_slicing_enabled, workload_from_env from accel_timeslicer.workload import SAMPLER_CLAIM, WorkloadRef, local_workload_name from server.store import RequestStore, StateStore, get_state_store, get_store +from server.telemetry import gpu_turn, observe_operation, op_finished, sampled_group, serve_metrics, serving, set_phase from server.vllm_options import cudagraph_capture_sizes, gpu_memory_utilization, sampler_batch_limits, split_stop, text_only_engine_kwargs tracer = trace.get_tracer("vllm.inference.worker") @@ -218,7 +219,7 @@ async def holding_gpu(time_slicer: TimeSlicerClient | None, workload: WorkloadRe if time_slicer is None: yield return - async with time_slicer.acquire(workload): + async with gpu_turn(time_slicer.acquire(workload)): if sampler is not None: await sampler.wake() try: @@ -232,13 +233,17 @@ async def process_batch(sampler: Sampler, store: RequestStore, requests: list[di """Serve consecutive requests that share a weights_path together, updating weights between groups.""" for weights_path, group in groupby(requests, key=lambda req: req.get("weights_path")): batch = list(group) + updating = time.time() try: await sampler.ensure_weights(weights_path) except Exception as exc: for request in batch: + op_finished(request, "sampler", request.get("model_id"), updating, type(exc).__name__) await store.set_result(request["request_id"], failed_response(f"vLLM weight update failed: {exc}")) continue + started = time.time() await serve_group(sampler, store, batch) + sampled_group(batch, started) async def serve_group(sampler: Sampler, store: RequestStore, batch: list[dict[str, Any]]) -> None: @@ -261,6 +266,8 @@ async def serve_group(sampler: Sampler, store: RequestStore, batch: list[dict[st async def fail_requests(store: RequestStore, requests: list[dict[str, Any]], error: Any) -> None: """Best effort, so clients see the failure instead of waiting forever.""" for request in requests: + # serve_group passes the engine's death as a message, not an exception. + op_finished(request, "sampler", request.get("model_id"), time.time(), type(error).__name__ if isinstance(error, Exception) else "EngineDied") try: await store.set_result(request["request_id"], failed_response(f"vLLM Worker Error: {error}")) except Exception as exc: @@ -270,7 +277,7 @@ async def fail_requests(store: RequestStore, requests: list[dict[str, Any]], err async def process_request(sampler: Sampler, store: RequestStore, request: dict[str, Any]) -> None: with tracer.start_as_current_span("process_sampling_request", context=propagate.extract(request.get("trace_context", {}))): try: - result = await sampler.generate(request) + result = await observe_operation(request, "sampler", request.get("model_id"), lambda: sampler.generate(request)) result["type"] = "sample" except Exception as exc: result = failed_response(f"vLLM Worker Error: {exc}") @@ -298,6 +305,7 @@ async def serve( sampler = Sampler(make_engine()) if time_slicer is not None: await sampler.sleep() # give the memory back before releasing the slot + set_phase("ready") state = get_state_store() try: try: @@ -355,6 +363,7 @@ async def serve_time_sliced(model_id: str, store: RequestStore, make_engine: Cal async def run_sampling_worker(model_id: str) -> None: fft_enabled = os.getenv("OPEN_RL_ENABLE_FFT", "").lower() == "true" engine_kwargs = engine_kwargs_from_env(fft_enabled) + serve_metrics() def make_engine() -> AsyncLLMEngine: return AsyncLLMEngine.from_engine_args(AsyncEngineArgs(**engine_kwargs)) @@ -371,10 +380,11 @@ def make_engine() -> AsyncLLMEngine: except NotImplementedError: break # An exclusive sampler is alone on its GPU, so it serves awake like a LoRA one. - if fft_enabled and time_slicing_enabled(): - await serve_time_sliced(model_id, get_store(), make_engine) - else: - await serve(model_id, get_store(), make_engine) + async with serving("sampler", model_id): + if fft_enabled and time_slicing_enabled(): + await serve_time_sliced(model_id, get_store(), make_engine) + else: + await serve(model_id, get_store(), make_engine) except asyncio.CancelledError: pass # serve has completed cleanup before cancellation reaches here. finally: diff --git a/src/server/worker_manager.py b/src/server/worker_manager.py index 3741e80b5..56539cd57 100644 --- a/src/server/worker_manager.py +++ b/src/server/worker_manager.py @@ -160,6 +160,10 @@ def inspect_owner(self, owner: str) -> list[dict[str, Any]]: `age_seconds` where the launcher knows them.""" ... + def workloads(self) -> list[dict[str, Any]]: + """The scheduler Workload objects this manager created; none for a launcher without a scheduler.""" + ... + class LocalWorkerManager: """Runs each runtime as a subprocess of the API server, for development.""" @@ -251,6 +255,9 @@ def inspect_owner(self, owner: str) -> list[dict[str, Any]]: ) return workers + def workloads(self) -> list[dict[str, Any]]: + return [] + def python_command(extras: list[str], module: str, args: list[str]) -> list[str]: if shutil.which("uv"): diff --git a/tests/test_api_server_paths.py b/tests/test_api_server_paths.py index 51a7d9b54..2ee422d04 100644 --- a/tests/test_api_server_paths.py +++ b/tests/test_api_server_paths.py @@ -7,9 +7,10 @@ from fastapi.testclient import TestClient -from server import api_server +from server import api_server, telemetry from server.store import InMemoryStateStore, InMemoryStore from server.worker_manager import LocalWorkerManager +from tests.api_client import asgi_client from tests.test_session_lifecycle import RuntimeManager @@ -20,6 +21,7 @@ class ApiServerTest(unittest.TestCase): def setUp(self) -> None: self.enterContext(patch.object(api_server, "store", InMemoryStore())) self.enterContext(patch.object(api_server, "state", InMemoryStateStore())) + self.enterContext(patch.object(api_server.session_registry, "state", api_server.state)) self.client = TestClient(api_server.app) def post(self, path: str, body: dict, **kwargs): @@ -33,6 +35,9 @@ def queued(self) -> list[dict]: class GetInfoTest(ApiServerTest): + def test_default_gateway_does_not_expose_fleet_inspection(self) -> None: + self.assertEqual(self.client.get("/api/v1/inspect/inventory").status_code, 404) + def test_get_info_uses_base_model_env(self) -> None: with patch.dict(os.environ, {"BASE_MODEL": "env-model"}, clear=True): info = self.post("get_info", {"model_id": "model-a"}).json() @@ -571,5 +576,37 @@ def test_sampler_replicas_need_a_server_that_launches_pods(self) -> None: self.assertIn("openrl.sampler_replicas above 1", response.json()["error"]) +class TelemetryRecordsTest(ApiServerTest): + """Records are written in the background, so each test runs its requests in one loop and lets the writes land.""" + + def setUp(self) -> None: + super().setUp() + self.enterContext(patch("server.worker_manager.get_state_store", return_value=api_server.state)) + self.enterContext(patch.object(telemetry, "get_state_store", return_value=api_server.state)) + + def call(self, *requests: tuple[str, dict]) -> list[dict]: + async def run() -> list[dict]: + async with asgi_client() as client: + answers = [(await client.post(f"/api/v1/{path}", json=body)).json() for path, body in requests] + await asyncio.gather(*list(telemetry.writes)) + return answers + + return asyncio.run(run()) + + def test_a_shared_trainers_delete_is_counted_neither_enqueued_nor_popped(self) -> None: + [created] = self.call(("create_model", {"base_model": "m"})) + self.assertEqual(self.post("delete_model", {"model_id": created["request_id"]}).status_code, 200) + + async def trainer() -> None: + async with telemetry.serving("trainer", "m"): + requests = await api_server.store.get_requests(active_set_id="m-1") + self.assertEqual([r["op"] for r in requests], ["create_model", "delete_model"]) + telemetry.popped(requests) + + asyncio.run(trainer()) + counts = asyncio.run(api_server.state.read_counts(telemetry.key("ledger", created["request_id"]))) + self.assertEqual(counts, {"enqueued": 1, "popped": 1}) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py new file mode 100644 index 000000000..672e4ea8b --- /dev/null +++ b/tests/test_dashboard.py @@ -0,0 +1,48 @@ +"""The inspection routes as an agent calls them, over a snapshot with one run and its recorded optim_step.""" + +import asyncio +import unittest +from unittest import mock + +from fastapi.testclient import TestClient + +from server.dashboard import fleet, router +from server.dashboard.__main__ import app +from server.store import InMemoryStateStore +from server.telemetry import key + +RUN = {"run_id": "r1", "name": "Run one", "model": "Qwen/base", "runtime_id": "r1", "created_at": 0, "completed_at": None} +RUN |= {"lifecycle": "placed", "status": "active", "workloads": [], "pods": [], "settings": {}, "session_ids": []} + + +class ApiTest(unittest.TestCase): + def setUp(self): + self.store = InMemoryStateStore() + op = {"run_id": "r1", "role": "trainer", "pod": "trainer-0", "operation": "optim_step", "status": "succeeded", "started_at": 100, "at": 150} + asyncio.run(self.store.append_sample(key("operations", "r1"), op, 10, None)) + self.snapshot = {"observed_at": 200, "placements": [], "workers": [], "runs": [RUN], "nodes": [], "pods": [], "errors": []} + fleet.cache.clear() + self.enterContext(mock.patch.object(fleet, "snapshot", mock.AsyncMock(return_value=self.snapshot))) + self.enterContext(mock.patch("server.dashboard.facts.get_state_store", return_value=self.store)) + self.enterContext(mock.patch.object(fleet, "get_state_store", return_value=self.store)) + self.enterContext(mock.patch.object(router, "query", mock.AsyncMock(return_value={"series": []}))) # Prometheus has no series + self.client = TestClient(app) + + def test_routes_are_read_only_and_a_run_summary_reads_the_store(self): + summary = self.client.get("/api/v1/inspect/runs/r1/summary").json() + self.assertEqual((summary["steps"], summary["last_op"]["operation"]), (1, "optim_step")) + self.assertEqual(self.client.get("/api/v1/inspect/runs/missing/summary").status_code, 404) + self.assertTrue(all(set(methods) == {"get"} for methods in self.client.get("/openapi.json").json()["paths"].values())) + + def test_a_session_id_names_its_one_run_and_a_session_with_several_runs_is_refused(self): + self.snapshot["runs"] = [{**RUN, "session_ids": ["s1", "s2"]}, {**RUN, "run_id": "r2", "runtime_id": "r2", "session_ids": ["s2"]}] + by_session = self.client.get("/api/v1/inspect/runs/s1/summary").json() + self.assertEqual((by_session["run_id"], by_session["steps"]), ("r1", 1)) + refused = self.client.get("/api/v1/inspect/runs/s2/summary") + self.assertEqual(refused.status_code, 409) + self.assertIn("r1, r2", refused.text) + self.assertEqual(self.client.get("/api/v1/inspect/runs/s3/summary").status_code, 404) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_dashboard_facts.py b/tests/test_dashboard_facts.py new file mode 100644 index 000000000..b38ef2917 --- /dev/null +++ b/tests/test_dashboard_facts.py @@ -0,0 +1,104 @@ +"""Run facts replay recorded incidents: each incident must show up as a fact an agent can read without a verdict.""" + +import asyncio +import unittest +from unittest import mock + +from server.dashboard import facts, fleet +from server.dashboard.facts import request_queue, sampler_queue +from server.store import InMemoryStateStore +from server.telemetry import key + +NOW = 100_000.0 + + +def op(at, operation, status="succeeded", role="trainer", **fields): + return {"at": at, "started_at": at - 10, "operation": operation, "status": status, "role": role, "elapsed_seconds": 10.0, "metrics": {}, **fields} + + +def healthy_steps(count=5, every=600.0): + return [op(NOW - (count - i) * every, "optim_step", metrics={"grad_norm:mean": 1.0 + i}) for i in range(count)] + + +def worker(started_at, heartbeat_at, **fields): + return {"id": str(started_at), "pod": "t-0", "role": "trainer", "runtime_id": "r", "started_at": started_at, "heartbeat_at": heartbeat_at, **fields} + + +class RunFactsTest(unittest.TestCase): + """Records go into a real in-memory store, and the facts are read back through the same context the routes use.""" + + def setUp(self): + self.enterContext(mock.patch.object(facts, "get_state_store", side_effect=lambda: self.store)) + self.enterContext(mock.patch.object(fleet, "get_state_store", side_effect=lambda: self.store)) + + def facts_of(self, ops=(), workers=(), ledger=None, queued=0, lifecycle="placed", pods=(), samples_queued=0, completed_at=None): + self.store = InMemoryStateStore() + fleet.cache.clear() + + async def read(): + for record in ops: + await self.store.append_sample(key("operations", "r", *(["samples"] if record["operation"] == "sample" else [])), record, 10_000, None) + for event, value in (ledger or {}).items(): + for _ in range(value): + await self.store.increment_field(key("ledger", "r"), event, None) + queues = {request_queue("r"): queued, sampler_queue("r"): samples_queued} # the in-memory store keeps no queues + self.store.list_length = mock.AsyncMock(side_effect=queues.get) + run = {"run_id": "r", "runtime_id": "r", "lifecycle": lifecycle, "created_at": 0.0, "completed_at": completed_at, "workloads": []} + run.update(pods=list(pods), settings={"openrl.fine_tuning_type": "lora", "openrl.trainer_gpus": 1}) + ctx = await facts.context({"observed_at": NOW, "workers": list(workers), "placements": [], "runs": [run]}, run) + return ctx, facts.summary(ctx), facts.line(ctx) + + return asyncio.run(read()) + + def test_requests_held_by_a_trainer_that_stopped_reporting_name_the_dead_holder(self): # I1 + held = {"op": "forward_backward", "run_id": "r", "count": 3, "since": NOW - 207360} + _, summary, line = self.facts_of(healthy_steps(), workers=[worker(0, NOW - 600, pod="harvey-trainer-0", outstanding=[held])]) + (request,) = summary["requests"]["unanswered"] + self.assertEqual((request["pod"], request["holder_alive"], request["age_s"]), ("harvey-trainer-0", False, 207360)) + self.assertEqual(line["held_by_dead_worker"], 3) + + def test_a_request_lost_between_push_and_pop_shows_in_the_ledger_and_queued_samples_show_too(self): # I4 + counts = {"enqueued": 11, "popped": 10, "answered": 10} + _, lost, _ = self.facts_of(healthy_steps(), ledger=counts) + self.assertEqual(lost["requests"]["ledger"]["lost"], 1) + _, queued, line = self.facts_of(healthy_steps(), ledger=counts, queued=1, samples_queued=64) + self.assertEqual((queued["requests"]["ledger"]["lost"], queued["requests"]["samples_queued"], line["samples_queued"]), (0, 64, 64)) + + def test_a_nan_gradient_names_the_op_step_and_metric(self): # I30 + _, summary, _ = self.facts_of([*healthy_steps(), op(NOW - 5, "optim_step", nonfinite=["grad_norm:mean"], step=5)]) + first = summary["optimizer"]["nonfinite"]["first"] + self.assertEqual((first["operation"], first["step"], first["metrics"]), ("optim_step", 5, ["grad_norm:mean"])) + + def test_a_run_that_ended_on_a_failed_create_model_names_the_error_and_where_to_read_it(self): + _, summary, _ = self.facts_of([op(NOW - 600, "create_model", status="failed", error_type="ValueError", pod="t-0")], lifecycle="ended") + self.assertEqual( + (summary["steps"], summary["last_op"]["operation"], summary["failures"]["ops"][0]["error_type"]), (0, "create_model", "ValueError") + ) + logs = summary["failures"]["logs"] + self.assertEqual((logs["read"], logs["args"]["pod"], logs["args"]["q"]), ("logs", "t-0", "ValueError")) + + def test_relaunch_rules(self): + first, after = worker(NOW - 1000, NOW - 905, pod_uid="u1"), {"started_at": NOW - 880, "heartbeat_at": NOW} + holding = [{"op": "create_model", "run_id": "r", "count": 1, "since": NOW - 950}] + cases = { + "container restarted in place": ([first, {**first, **after}], None, 1), + "new pod after one died holding work": ([{**first, "outstanding": holding}, {**first, **after, "pod_uid": "u2"}], None, 1), + "clean exit": ([{**first, "phase": "exited"}, {**first, **after}], None, 0), + "idle pod recreated, as a shared runtime's teardown does": ([first, {**first, **after, "pod_uid": "u2"}], None, 0), + "next run on the shared runtime, same pod name": ([first, {**first, "started_at": NOW - 300, "heartbeat_at": NOW}], NOW - 500, 0), + } + for name, (workers, completed_at, expected) in cases.items(): + with self.subTest(name): + _, summary, line = self.facts_of(workers=workers, completed_at=completed_at) + self.assertEqual((line["relaunches"], "relaunches" in summary), (expected, bool(expected))) + + def test_an_oom_killed_container_is_an_incident_and_a_healthy_run_has_no_failure_blocks(self): + oom = {"at": NOW - 60, "pod": "t-0", "reason": "OOMKilled", "message": "worker exited with code 137"} + _, summary, _ = self.facts_of(healthy_steps(), pods=[{"name": "t-0", "incidents": [oom]}]) + self.assertEqual(summary["incidents"][0]["reason"], "OOMKilled") + _, healthy, _ = self.facts_of(healthy_steps(), ledger={"enqueued": 5, "popped": 5, "answered": 5}) + self.assertFalse({"failures", "relaunches", "incidents", "samplers"} & set(healthy)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fft_batch_failure.py b/tests/test_fft_batch_failure.py index 962393168..ccaf32133 100644 --- a/tests/test_fft_batch_failure.py +++ b/tests/test_fft_batch_failure.py @@ -1,10 +1,12 @@ import asyncio import contextlib +import json import os import unittest from unittest.mock import patch -from server.store import InMemoryStore +from server import telemetry +from server.store import InMemoryStateStore, InMemoryStore from server.training_requests_processor import TrainingRequestsProcessor @@ -85,5 +87,46 @@ async def handled(request): self.assertEqual(exits, [False]) +class FailedAnswerRecordTest(unittest.TestCase): + """Every failed answer a client gets has one failed record, including failures before the op runs.""" + + def setUp(self): + self.state = InMemoryStateStore() + self.enterContext(patch.object(telemetry, "get_state_store", return_value=self.state)) + + def test_a_wake_or_parse_failure_records_every_request_it_failed(self) -> None: + wake = processor(BatchStore([{"request_id": "fb-1", "op": "forward_backward", "model_id": "run-a"}]), SlicerStub()) + parse = processor(BatchStore([{"request_id": "bad-1", "op": "optim_step", "model_id": "run-a", "adam_params": "not a dict"}]), None) + + async def serve(): + async with telemetry.serving("trainer", "run-a"): + for proc in (wake, parse): + with contextlib.suppress(RuntimeError): + await proc.run_once() + return await self.state.read_samples(telemetry.key("operations", "run-a")) + + records = [(r["request_id"], r["operation"], r["status"]) for r in asyncio.run(serve())] + self.assertEqual(records, [("fb-1", "forward_backward", "failed"), ("bad-1", "optim_step", "failed")]) + + def test_an_exit_after_shutdown_says_the_process_exited_and_lands_its_writes(self) -> None: + proc = processor(BatchStore([{"request_id": "s-1", "op": "shutdown_workers", "model_id": "run-a"}]), SlicerStub()) + increment = self.state.increment_field + + async def slow_increment(*args): + await asyncio.sleep(0.05) + return await increment(*args) + + async def serve(): + async with telemetry.serving("trainer", "run-a") as worker: + telemetry.record("run-a", "popped") # still in flight when the process exits + with patch("os._exit", side_effect=SystemExit), self.assertRaises(SystemExit): + await proc.run_once() + record = json.loads(await self.state.get_value(telemetry.key("worker", worker.record["id"])) or "{}") + return record.get("phase"), (await self.state.read_counts(telemetry.key("ledger", "run-a"))).get("popped") + + with patch.object(self.state, "increment_field", slow_increment): + self.assertEqual(asyncio.run(serve()), ("exited", 1)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_sampler_patch.py b/tests/test_sampler_patch.py index a27bc16f0..8e89f3ac7 100644 --- a/tests/test_sampler_patch.py +++ b/tests/test_sampler_patch.py @@ -9,6 +9,8 @@ from unittest.mock import AsyncMock, Mock, patch from accel_timeslicer.workload import WorkloadRef +from server import telemetry +from server.store import InMemoryStateStore from server.vllm_sampler import SHUTDOWN_SENTINEL, Sampler, process_batch, serve, serve_time_sliced @@ -135,7 +137,9 @@ async def asyncSetUp(self): patch("server.vllm_sampler.workload_from_env", return_value=self.workload), patch("server.vllm_sampler.get_state_store", return_value=self.store), patch("server.vllm_sampler.ENGINE_POLL_SECONDS", 0.01), + patch.object(telemetry, "get_state_store", return_value=(state := InMemoryStateStore())), ] + self.state = state for p in self.patches: p.start() self.addCleanup(p.stop) @@ -216,10 +220,11 @@ async def hang(request_id): await asyncio.Event().wait() self.engine.generate.side_effect = make_engine(hang).generate - self.store.get_sampling_requests_for_model.return_value = [{"request_id": "1"}] + self.store.get_sampling_requests_for_model.return_value = [{"request_id": "1", "model_id": "run"}] with self.assertRaisesRegex(RuntimeError, "engine is dead"): await serve_time_sliced("test", self.store, self.factory) self.assertIn("engine died", self.results()["1"]["error_message"]) + self.assertEqual(await self.recorded(), [("1", "failed", "EngineDied")]) self.engine.shutdown.assert_called_once() self.slicer.unregister.assert_awaited_once() @@ -239,6 +244,14 @@ async def faulting_slot(workload): self.slicer.unregister.assert_not_called() self.slicer.close.assert_awaited_once() + async def test_failed_weight_update_keeps_a_failed_record(self): + self.engine.update_weights.side_effect = RuntimeError("CUDA out of memory") + request = {"request_id": "1", "model_id": "run", "weights_path": "a"} + self.store.get_sampling_requests_for_model.side_effect = [[request, {"request_id": SHUTDOWN_SENTINEL}]] + await serve_time_sliced("test", self.store, self.factory) + self.assertIn("out of memory", self.results()["1"]["error_message"]) + self.assertEqual(await self.recorded(), [("1", "failed", "RuntimeError")]) + async def test_loop_errors_fail_unserved_requests_and_retry(self): self.store.get_sampling_requests_for_model.side_effect = [ RuntimeError("redis down"), @@ -251,6 +264,11 @@ async def test_loop_errors_fail_unserved_requests_and_retry(self): self.assertEqual(self.results()["2"]["type"], "sample") self.slicer.unregister.assert_awaited_once() + async def recorded(self) -> list[tuple[str, str, str]]: + """Every failed answer a client gets has one sample record.""" + await asyncio.gather(*list(telemetry.writes)) + return [(r["request_id"], r["status"], r["error_type"]) for r in await self.state.read_samples(telemetry.key("operations", "run", "samples"))] + if __name__ == "__main__": unittest.main() diff --git a/tests/test_session_lifecycle.py b/tests/test_session_lifecycle.py index d6fb8e349..b9af376d8 100644 --- a/tests/test_session_lifecycle.py +++ b/tests/test_session_lifecycle.py @@ -23,6 +23,9 @@ def ensure(self, model_id, role): def release(self, model_id): self.released_models.append(model_id) + def workloads(self): + return [] + def release_owner(self, owner): self.released.append(owner) return {"test-base"} if owner == "test-base" else set() diff --git a/tests/test_telemetry_faults.py b/tests/test_telemetry_faults.py new file mode 100644 index 000000000..5fb65e592 --- /dev/null +++ b/tests/test_telemetry_faults.py @@ -0,0 +1,100 @@ +"""Telemetry must never change what a worker does. Each test serves the same work with a healthy telemetry store and with +one that fails, and expects the same answers, in about the same time.""" + +import asyncio +import os +import threading +import time +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from server import telemetry +from server.store import InMemoryStateStore +from server.training_requests_processor import TrainingRequestsProcessor +from tests.test_fft_batch_failure import BatchStore, SlicerStub + +try: + from server import vllm_sampler + from tests.test_sampler_patch import make_engine +except ImportError: # vLLM is a GPU extra + vllm_sampler = None + +BOUND = 1.0 # a hung write is dropped after WRITE_SECONDS; the serve must finish well inside it + + +class FaultyStore: + """Every call raises, as with Redis down, or waits forever, as with a Redis that never answers.""" + + def __init__(self, hangs: bool): + self.hangs = hangs + + def __getattr__(self, name): + if not self.hangs: + raise ConnectionError(f"telemetry store {name} failed") + if name.endswith("_sync"): + return lambda *args, **kwargs: threading.Event().wait() + return lambda *args, **kwargs: asyncio.Event().wait() + + +def no_store(): + raise RuntimeError("no telemetry store configured") + + +FAULTS = {"raises": lambda: FaultyStore(False), "hangs": lambda: FaultyStore(True), "is missing": no_store} + + +class TelemetryFaultTest(unittest.TestCase): + def setUp(self) -> None: + self.enterContext(patch.object(telemetry, "WRITE_SECONDS", 5 * BOUND)) + self.enterContext(patch.object(telemetry, "FINISH_SECONDS", BOUND)) + self.enterContext(patch.dict(telemetry.run_steps, clear=True)) + + def assert_unchanged(self, work) -> None: + async def serve(): + async with telemetry.serving("trainer", "run-a"): + started = time.monotonic() + result = await asyncio.wait_for(work(), 2 * BOUND) + exiting = time.monotonic() + return result, exiting - started, time.monotonic() - exiting + + healthy = InMemoryStateStore() + with patch.object(telemetry, "get_state_store", lambda: healthy): + expected, _, _ = asyncio.run(serve()) + self.assertTrue(expected) + self.assertTrue(healthy.samples or healthy.get_value_sync(telemetry.key("sampled", "run-a")), "the healthy serve recorded its ops") + for fault, store in FAULTS.items(): + with self.subTest(store=fault), patch.object(telemetry, "get_state_store", store): + result, took, exiting = asyncio.run(serve()) + self.assertEqual(result, expected) + self.assertLess(took, BOUND) + self.assertLess(exiting, 2 * BOUND) + + def test_a_time_sliced_training_request_is_answered_the_same(self) -> None: + trainer = Mock(cpu_offload=True, optim_step=lambda adam_params, model_id: {"metrics": {"grad_norm:mean": 0.5}}) + + async def train(): + store = BatchStore([{"request_id": f"os-{i}", "op": "optim_step", "model_id": "run-a", "adam_params": {}} for i in range(2)]) + with patch.dict(os.environ, {"REDIS_URL": "redis://test"}): + processor = TrainingRequestsProcessor(store, trainer, "run-a", time_slicer=SlicerStub()) + await processor.run_once() + return store.futures + + self.assert_unchanged(train) + + @unittest.skipIf(vllm_sampler is None, "needs vLLM") + def test_a_sample_request_is_answered_the_same(self) -> None: + async def sample(): + store = AsyncMock() + store.get_sampling_requests_for_model.side_effect = [ + [{"request_id": "s-1", "model_id": "run-a", "prompt_token_ids": [1, 2], "max_tokens": 1}], + [{"request_id": vllm_sampler.SHUTDOWN_SENTINEL}], + ] + with patch.object(vllm_sampler, "get_state_store", return_value=AsyncMock()): + await vllm_sampler.serve("run-a", store, make_engine, SlicerStub(), vllm_sampler.WorkloadRef("sampler-run-a")) + return {call.args[0]: call.args[1] for call in store.set_result.call_args_list} + + self.assert_unchanged(sample) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_worker_record.py b/tests/test_worker_record.py new file mode 100644 index 000000000..f010499bb --- /dev/null +++ b/tests/test_worker_record.py @@ -0,0 +1,108 @@ +import asyncio +import json +import time +import unittest +from unittest import mock + +from server import telemetry +from server.store import InMemoryStateStore + + +def call(result=None, error=None): + return mock.AsyncMock(return_value=result or {}, side_effect=error) + + +class WorkerRecordTest(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self.store = InMemoryStateStore() + self.enterContext(mock.patch.object(telemetry, "get_state_store", return_value=self.store)) + self.enterContext(mock.patch.dict("os.environ", {"POD_NAME": "pod", "POD_UID": "uid", "NODE_NAME": "node"})) + self.enterContext(mock.patch.dict(telemetry.run_steps, clear=True)) + + async def saved(self, worker_id: str) -> dict: + await asyncio.gather(*list(telemetry.writes)) + return json.loads(await self.store.get_value(telemetry.key("worker", worker_id))) + + async def recorded(self, *parts: str) -> list[dict]: + await asyncio.gather(*list(telemetry.writes)) # writes are fire-and-forget + return await self.store.read_samples(telemetry.key("operations", *parts)) + + async def test_the_record_shows_settings_and_a_clean_exit(self) -> None: + env = {"OPEN_RL_FINE_TUNING_TYPE": "lora", "OPEN_RL_API_KEY": "sk-private", "HOME": "/root"} + with mock.patch.dict("os.environ", env): + async with telemetry.serving("trainer", "run-a") as worker: + record = await self.saved(worker.record["id"]) + self.assertTrue(telemetry.alive(record, record["heartbeat_at"])) + self.assertEqual(record["config"]["env"], {"OPEN_RL_FINE_TUNING_TYPE": "lora"}) + record = await self.saved(worker.record["id"]) + self.assertEqual(record["phase"], "exited") + self.assertFalse(telemetry.alive(record, record["heartbeat_at"])) + self.assertNotIn("sk-private", json.dumps(record)) + + async def test_the_record_stays_fresh_while_model_loading_blocks_the_event_loop(self) -> None: + with mock.patch.object(telemetry, "HEARTBEAT_SECONDS", 0.02): + async with telemetry.serving("trainer", "run-a") as worker: + blocked_at = time.time() + time.sleep(0.3) # loading weights holds the loop, and the heartbeat must not need it + record = json.loads(self.store.get_value_sync(telemetry.key("worker", worker.record["id"]))) + self.assertGreater(record["heartbeat_at"] - blocked_at, 0.2) + + async def test_popped_requests_stay_outstanding_and_are_counted_until_answered(self) -> None: + async with telemetry.serving("trainer", "base") as worker: + telemetry.popped([{"request_id": f"f{i}", "op": "forward_backward", "model_id": "run-a"} for i in range(3)] + [{"op": "no id"}]) + telemetry.popped([{"request_id": "s1", "model_id": "base", "logical_run_id": "run-b"}]) + # A shared trainer's delete_model was never counted as enqueued, so it is not outstanding either. + telemetry.popped([{"request_id": "d1", "op": "delete_model", "model_id": "run-a"}]) + telemetry.answered("f0") + groups = {(g["op"], g["run_id"]): g["count"] for g in json.loads(worker.snapshot())["outstanding"]} + self.assertEqual(groups, {("forward_backward", "run-a"): 2, ("unknown", "run-b"): 1}) + self.assertEqual(await self.store.read_counts(telemetry.key("ledger", "run-a")), {"popped": 3, "answered": 1}) + + async def test_an_op_raises_as_it_would_and_only_rank_zero_records_its_failure(self) -> None: + request = {"model_id": "run-a", "op": "train", "payload": "private input"} + with self.assertRaisesRegex(ValueError, "private error"): + await telemetry.observe_operation(request, "trainer", "run-a", call(error=ValueError("private error"))) + with self.assertRaises(asyncio.CancelledError): + await telemetry.observe_operation(request, "trainer", "run-a", call(error=asyncio.CancelledError())) + with mock.patch.dict("os.environ", {"RANK": "1"}): + await telemetry.observe_operation(request, "trainer", "run-a", call()) + (sample,) = await self.recorded("run-a") + self.assertEqual((sample["status"], sample["error_type"]), ("failed", "ValueError")) + self.assertNotIn("private", repr(sample)) + + async def test_trainer_ops_carry_their_step_and_nonfinite_metrics_and_sampling_never_evicts_them(self) -> None: + await self.store.set_value(telemetry.key("steps", "r"), "5") # a restarted trainer resumes the run's count + failed, nan = {"type": "RequestFailedResponse"}, {"metrics": {"loss": 1.5, "grad_norm": float("nan"), "kl": float("inf")}} + for op, result in (("forward_backward", nan), ("optim_step", None), ("optim_step", failed), ("forward_backward", None)): + await telemetry.observe_operation({"model_id": "r", "op": op}, "trainer", "r", call(result)) + await telemetry.observe_operation({"model_id": "r", "op": "forward_backward", "forward_only": True}, "trainer", "r", call()) + with mock.patch.object(telemetry, "SAMPLE_LIMIT", 3): + for i in range(6): # a shared sampler's requests name the base model's queue and the run they sample for + request = {"op": "sample", "model_id": "base", "logical_run_id": "r", "request_id": f"s{i}"} + telemetry.op_finished(request, "sampler", "base", 0.0, "EngineDied" if i % 2 else None) + trainer = await self.recorded("r") + self.assertEqual( + [(s["operation"], s["status"], s["step"]) for s in trainer], + [("forward_backward", "succeeded", 5), ("optim_step", "succeeded", 5), ("optim_step", "failed", 6)] + + [("forward_backward", "succeeded", 6), ("forward", "succeeded", 6)], + ) + self.assertEqual((trainer[0]["metrics"], trainer[0]["nonfinite"]), ({"loss": 1.5}, ["grad_norm", "kl"])) # JSON has no NaN + self.assertEqual(await self.store.get_value(telemetry.key("steps", "r")), "6") + + +if __name__ == "__main__": + unittest.main() + + +class PlacementRecordTest(unittest.TestCase): + def test_a_placement_keeps_its_pods_termination_after_the_pod_status_drops_it(self) -> None: + workload = { + "metadata": {"name": "w"}, + "spec": {"role": "trainer", "modelID": "m"}, + "status": {"placedAt": "2026-10-21T06:00:00Z", "podName": "p"}, + } + oom = {"at": telemetry.epoch("2026-10-21T06:30:00Z"), "container": "worker", "reason": "OOMKilled", "exit_code": 137, "signal": None} + first = telemetry.merge_placements([], {"w": telemetry.placement(workload, {"p": [oom]})}, time.time()) + later = telemetry.merge_placements(first, {"w": telemetry.placement(workload, {})}, time.time()) + self.assertEqual(later[0]["terminations"], [oom]) + self.assertIsNone(later[0]["released_at"])