diff --git a/Cargo.lock b/Cargo.lock index ce6bbec32..2a432feeb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5529,6 +5529,7 @@ dependencies = [ "tower-http 0.7.0", "tracing", "tracing-subscriber", + "url", ] [[package]] @@ -5539,6 +5540,7 @@ dependencies = [ "common", "reqwest 0.13.4", "serde", + "serde_json", "tracing", ] diff --git a/ceres/src/model/orion_runner.rs b/ceres/src/model/orion_runner.rs index b1237dbe8..6539e3f58 100644 --- a/ceres/src/model/orion_runner.rs +++ b/ceres/src/model/orion_runner.rs @@ -5,6 +5,9 @@ use utoipa::ToSchema; pub struct StartRunnerRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub target: Option, + /// Force recreate when a Running VM already exists for this mono's domain. + #[serde(default)] + pub replace: bool, #[serde(default, skip_serializing_if = "Option::is_none")] pub image_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -23,6 +26,8 @@ pub struct StartRunnerRequest { pub struct StartRunnerResponse { pub vm_id: String, pub phase: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub domain: Option, } #[derive(Serialize, Deserialize, ToSchema, Debug, Clone)] @@ -30,6 +35,8 @@ pub struct RunnerStatusResponse { pub vm_id: String, pub phase: String, #[serde(default, skip_serializing_if = "Option::is_none")] + pub domain: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub vm_ip: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub log_file: Option, diff --git a/clients/orion-scheduler-client/Cargo.toml b/clients/orion-scheduler-client/Cargo.toml index 555d120a4..5ddaf1eb9 100644 --- a/clients/orion-scheduler-client/Cargo.toml +++ b/clients/orion-scheduler-client/Cargo.toml @@ -13,4 +13,5 @@ common = { workspace = true } reqwest = { workspace = true, features = ["json"] } anyhow = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } tracing = { workspace = true } diff --git a/clients/orion-scheduler-client/src/http_client.rs b/clients/orion-scheduler-client/src/http_client.rs index cfbd3477a..91a4bb988 100644 --- a/clients/orion-scheduler-client/src/http_client.rs +++ b/clients/orion-scheduler-client/src/http_client.rs @@ -59,7 +59,8 @@ impl OrionSchedulerHttpClient { let res = self.auth_headers(req).send().await?; let status = res.status(); let body: StartRunnerSchedulerResponse = res.json().await?; - if status.is_success() || status.as_u16() == 202 { + // 200 OK (idempotent), 202 Accepted (provisioning), 409 Conflict + if status.is_success() || status.as_u16() == 202 || status.as_u16() == 409 { Ok(body) } else { Err(anyhow::anyhow!( @@ -70,12 +71,56 @@ impl OrionSchedulerHttpClient { } } + pub async fn get_vm_status(&self, vm_id: &str) -> anyhow::Result { + let url = format!("{}/vms/{}", self.base_url, vm_id); + let req = self.client.get(&url).timeout(Duration::from_secs(30)); + let res = self.auth_headers(req).send().await?; + let status = res.status(); + if status.as_u16() == 404 { + return Ok(SchedulerStatusResponse { + status: "no_vm".to_string(), + phase: Some("no_vm".to_string()), + vm_id: Some(vm_id.to_string()), + domain: None, + vm_ip: None, + uptime_secs: None, + log_file: None, + error: Some("VM not found".to_string()), + }); + } + if status.is_success() { + Ok(res.json().await?) + } else { + Err(anyhow::anyhow!( + "Scheduler get_vm_status failed: {}", + status + )) + } + } + pub async fn get_status(&self) -> anyhow::Result { let url = format!("{}/status", self.base_url); let req = self.client.get(&url).timeout(Duration::from_secs(30)); let res = self.auth_headers(req).send().await?; if res.status().is_success() { - Ok(res.json().await?) + // List form — not used by mono GET by id path anymore. + let v: serde_json::Value = res.json().await?; + if let Some(vms) = v.get("vms").and_then(|x| x.as_array()) { + if let Some(first) = vms.first() { + return Ok(serde_json::from_value(first.clone())?); + } + return Ok(SchedulerStatusResponse { + status: "no_vm".to_string(), + phase: Some("no_vm".to_string()), + vm_id: None, + domain: None, + vm_ip: None, + uptime_secs: None, + log_file: None, + error: None, + }); + } + Ok(serde_json::from_value(v)?) } else { Err(anyhow::anyhow!( "Scheduler get_status failed: {}", diff --git a/clients/orion-scheduler-client/src/lib.rs b/clients/orion-scheduler-client/src/lib.rs index 2dca3486a..fc5bcf323 100644 --- a/clients/orion-scheduler-client/src/lib.rs +++ b/clients/orion-scheduler-client/src/lib.rs @@ -1,4 +1,4 @@ -//! HTTP client for orion-scheduler VM provisioning (`/webhook`, `/status`). +//! HTTP client for orion-scheduler VM provisioning (`/webhook`, `/status`, `/vms/{id}`). mod http_client; @@ -11,6 +11,8 @@ use serde::{Deserialize, Serialize}; pub struct StartRunnerPayload { #[serde(skip_serializing_if = "Option::is_none")] pub target: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub replace: bool, pub server_ws: String, pub scorpio_base_url: String, pub scorpio_lfs_url: String, @@ -28,17 +30,21 @@ pub struct StartRunnerPayload { pub image_memory_mb: Option, } -/// Response from scheduler `POST /webhook` (async 202 or sync 200). +/// Response from scheduler `POST /webhook` (async 202, sync 200, conflict 409). #[derive(Debug, Clone, Deserialize)] pub struct StartRunnerSchedulerResponse { pub status: String, pub vm_id: Option, + #[serde(default)] + pub domain: Option, + #[serde(default)] + pub phase: Option, pub error: Option, #[serde(default)] pub orion_log_file: Option, } -/// Response from scheduler `GET /status`. +/// Response from scheduler `GET /vms/{id}` or filtered `/status`. #[derive(Debug, Clone, Deserialize)] pub struct SchedulerStatusResponse { pub status: String, @@ -46,6 +52,8 @@ pub struct SchedulerStatusResponse { pub phase: Option, pub vm_id: Option, #[serde(default)] + pub domain: Option, + #[serde(default)] pub vm_ip: Option, #[serde(default)] pub uptime_secs: Option, @@ -79,6 +87,11 @@ impl OrionSchedulerClient { self.http.start_runner(payload).await } + pub async fn get_vm_status(&self, vm_id: &str) -> anyhow::Result { + self.http.get_vm_status(vm_id).await + } + + /// Backward-compatible list/status endpoint. pub async fn get_status(&self) -> anyhow::Result { self.http.get_status().await } diff --git a/mono/src/api/router/orion_runner_router.rs b/mono/src/api/router/orion_runner_router.rs index c93df70b8..15db2850a 100644 --- a/mono/src/api/router/orion_runner_router.rs +++ b/mono/src/api/router/orion_runner_router.rs @@ -140,6 +140,7 @@ async fn start_runner( let payload = StartRunnerPayload { target: req.target, + replace: req.replace, server_ws: env.server_ws, scorpio_base_url: env.scorpio_base_url, scorpio_lfs_url: env.scorpio_lfs_url, @@ -158,6 +159,17 @@ async fn start_runner( ) })?; + if sched_resp.status == "conflict" { + return Err(ApiError::with_status( + StatusCode::CONFLICT, + anyhow!( + "Runner already provisioning for domain {:?}: {}", + sched_resp.domain, + sched_resp.error.unwrap_or_else(|| "conflict".to_string()) + ), + )); + } + let vm_id = sched_resp.vm_id.ok_or_else(|| { ApiError::with_status( StatusCode::BAD_GATEWAY, @@ -170,17 +182,20 @@ async fn start_runner( ) })?; - let phase = if sched_resp.status == "provisioning" { - "provisioning".to_string() - } else if sched_resp.status == "ok" { - "running".to_string() - } else { - sched_resp.status - }; + let phase = sched_resp.phase.unwrap_or_else(|| { + if sched_resp.status == "provisioning" { + "provisioning".to_string() + } else if sched_resp.status == "ok" { + "running".to_string() + } else { + sched_resp.status + } + }); Ok(Json(CommonResult::success(Some(StartRunnerResponse { vm_id, phase, + domain: sched_resp.domain, })))) } @@ -209,25 +224,27 @@ async fn get_runner_status( ensure_admin(&state, &user).await?; let client = scheduler_client(&state)?; - let sched = client.get_status().await.map_err(|e| { + let sched = client.get_vm_status(&id).await.map_err(|e| { ApiError::with_status( StatusCode::BAD_GATEWAY, anyhow!("Scheduler request failed: {}", e), ) })?; - if sched.vm_id.as_deref() != Some(id.as_str()) { - return Err(ApiError::not_found(anyhow!("Runner VM '{}' not found", id))); - } - let phase = sched .phase + .clone() .or_else(|| Some(sched.status.clone())) .unwrap_or_else(|| "unknown".to_string()); + if phase == "no_vm" || sched.vm_id.as_deref() != Some(id.as_str()) { + return Err(ApiError::not_found(anyhow!("Runner VM '{}' not found", id))); + } + Ok(Json(CommonResult::success(Some(RunnerStatusResponse { vm_id: id, phase, + domain: sched.domain, vm_ip: sched.vm_ip, log_file: sched.log_file, error: sched.error, diff --git a/moon/apps/web/hooks/OrionClient/usePostStartRunner.ts b/moon/apps/web/hooks/OrionClient/usePostStartRunner.ts index 22578eb44..486778571 100644 --- a/moon/apps/web/hooks/OrionClient/usePostStartRunner.ts +++ b/moon/apps/web/hooks/OrionClient/usePostStartRunner.ts @@ -1,23 +1,27 @@ import { useMutation } from '@tanstack/react-query' import { toast } from 'react-hot-toast' -import type { StartRunnerResponse } from '@gitmono/types/generated' +import type { StartRunnerRequest, StartRunnerResponse } from '@gitmono/types/generated' import { legacyApiClient } from '@/utils/queryClient' export function usePostStartRunner() { const mutation = legacyApiClient.v1.postApiOrionRunners() - return useMutation({ - mutationFn: async () => { - const result = await mutation.request({}) + return useMutation({ + mutationFn: async (body) => { + const result = await mutation.request(body ?? {}) if (!result.req_result || !result.data) { throw new Error(result.err_message || 'Failed to start runner') } return result.data }, - onSuccess: () => { - toast.success('Runner provisioning started') + onSuccess: (data) => { + if (data.phase === 'running') { + toast.success(data.domain ? `Runner already running (${data.domain})` : 'Runner already running') + } else { + toast.success(data.domain ? `Runner provisioning started (${data.domain})` : 'Runner provisioning started') + } }, onError: (error) => { toast.error(error?.message || 'Failed to start runner') diff --git a/moon/apps/web/pages/[org]/oc/index.tsx b/moon/apps/web/pages/[org]/oc/index.tsx index 9cb42dc39..ded0c3747 100644 --- a/moon/apps/web/pages/[org]/oc/index.tsx +++ b/moon/apps/web/pages/[org]/oc/index.tsx @@ -29,6 +29,7 @@ const OrionClientPage: PageWithLayout = () => { const [currentPage, setCurrentPage] = React.useState(1) const [activeVmId, setActiveVmId] = React.useState(null) const [activePhase, setActivePhase] = React.useState(null) + const [activeDomain, setActiveDomain] = React.useState(null) const perPage = 8 @@ -90,6 +91,9 @@ const OrionClientPage: PageWithLayout = () => { React.useEffect(() => { if (!runnerStatus) return setActivePhase(runnerStatus.phase) + if (runnerStatus.domain) { + setActiveDomain(runnerStatus.domain) + } }, [runnerStatus]) React.useEffect(() => { @@ -98,14 +102,21 @@ const OrionClientPage: PageWithLayout = () => { } }, [runnerStatus?.phase, handleRefresh]) - const handleStartRunner = React.useCallback(() => { - startRunner(undefined, { - onSuccess: (data) => { - setActiveVmId(data.vm_id) - setActivePhase(data.phase) - } - }) - }, [startRunner]) + const handleStartRunner = React.useCallback( + (replace = false) => { + startRunner( + { replace }, + { + onSuccess: (data) => { + setActiveVmId(data.vm_id) + setActivePhase(data.phase) + setActiveDomain(data.domain ?? null) + } + } + ) + }, + [startRunner] + ) React.useEffect(() => { mutate(requestPayload, { @@ -159,7 +170,7 @@ const OrionClientPage: PageWithLayout = () => { {isAdmin ? ( ) : null} diff --git a/moon/packages/types/generated.ts b/moon/packages/types/generated.ts index 5b55e409a..10ceef69e 100644 --- a/moon/packages/types/generated.ts +++ b/moon/packages/types/generated.ts @@ -5639,6 +5639,7 @@ export type ReviewersResponse = { } export type RunnerStatusResponse = { + domain?: string | null error?: string | null log_file?: string | null phase: string @@ -5722,10 +5723,13 @@ export type StartRunnerRequest = { image_memory_mb?: number | null image_path?: string | null image_url?: string | null + /** Force recreate when a Running VM already exists for this domain */ + replace?: boolean target?: string | null } export type StartRunnerResponse = { + domain?: string | null phase: string vm_id: string } diff --git a/orion-scheduler/Cargo.toml b/orion-scheduler/Cargo.toml index 95c014497..ec76d539b 100644 --- a/orion-scheduler/Cargo.toml +++ b/orion-scheduler/Cargo.toml @@ -18,6 +18,7 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } futures-util = { workspace = true } async-stream = { workspace = true } +url = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] qlean = { workspace = true } diff --git a/orion-scheduler/DESIGN.md b/orion-scheduler/DESIGN.md index 9f46eeaed..9762ca4f4 100644 --- a/orion-scheduler/DESIGN.md +++ b/orion-scheduler/DESIGN.md @@ -2,7 +2,7 @@ ## 1. 概述 -**目的**:orion-scheduler 是一个服务,接收 GitHub Actions 的 webhook 回调,使用 qlean/QEMU/KVM 管理 VM 生命周期,将 Orion 二进制文件和配置部署到 VM,并管理 Orion 服务。 +**目的**:orion-scheduler 是常驻服务,接收 GitHub Actions / mono 代理的 webhook,用 qlean/QEMU/KVM 按 **`server_ws` 主机名(domain)** 管理多台 microVM:注入 Orion 二进制与 runner 配置,并在 keep-alive 模式下保持 VM 运行以支持日志与调试。 **前提条件(AWS EC2 环境)**: @@ -40,22 +40,31 @@ aws ec2 start-instances --instance-id i-xxxxx **架构**: ``` -GitHub Actions --webhook--> orion-scheduler --qlean--> QEMU/KVM VM - | - +-- SSH/SFTP --> orion 二进制 + 配置 +GHA / mono --POST /webhook--> orion-scheduler(单进程) + | + +---------------+---------------+ + | | | + domain A domain B …(最多 max_vms) + microVM microVM + | | + SSH/SFTP SSH/SFTP + orion + 配置 orion + 配置 ``` +唯一性 key = `Url::parse(server_ws).host()`。同 domain 冲突策略见 README;进程退出时关闭**全部**跟踪中的 VM,不使用全局 `pkill qemu`。 + ## 2. 组件 | 组件 | 描述 | | ------------------- | -------------------------------------------------------------------------------------------------------------------- | -| `main.rs` | 使用 axum 的 HTTP 服务器入口,支持优雅关闭 | -| `handlers.rs` | HTTP 请求处理器:/webhook, /health, /status, /logs/orion/stream, /scorpio/status, /shutdown | -| `state.rs` | 用于跟踪 VM 生命周期的 AppState | +| `main.rs` | axum 入口;`LISTEN_ADDR`;信号时关停全部跟踪 VM + run-dir reap | +| `handlers.rs` | `/webhook`、`/health`、`/status`、`/vms/{id}`、`/logs/...`、`/scorpio/*`、`/shutdown`、`/shutdown/all` | +| `state.rs` | `HashMap`;全局 `update_lock`;`VmPhase` | | `vm_manager.rs` | VM 部署操作(上传文件、替换环境变量、启动服务) | -| `orion_deployer.rs` | Orion 部署编排,协调 VM 操作 | -| `config.rs` | 动态配置加载和管理,支持从 JSON 文件读取 `default_image` 与宿主机路径配置 | +| `orion_deployer.rs` | 按 domain 编排部署;`domain_from_server_ws`;仅关同 domain 旧实例 | +| `config.rs` | `target_config.json`:路径、`default_image`、可选 `max_vms` | | `keep_alive.rs` | Keep-alive VM 包装器,支持持久化 VM 连接 | +| `vm_cleanup.rs` | 按 `runs/*/qemu.pid` 杀进程与清理死目录(替代宿主机全局 pkill) | ## 3. API 端点 @@ -71,47 +80,64 @@ GitHub Actions --webhook--> orion-scheduler --qlean--> QEMU/KVM VM ### GET /status -获取当前 VM 状态。 - -**响应**(VM 运行中): +列出全部 VM,或按查询过滤。 -```json -{"status": "running", "vm_id": "orion-vm-1234567890", "vm_ip": "192.168.221.87", "uptime_secs": 60, "log_file": "/var/log/orion-scheduler/orion-vm-1234567890-1746766200.log"} +```bash +curl http://localhost:8080/status +curl 'http://localhost:8080/status?domain=orion.gitmega.com' +curl 'http://localhost:8080/status?vm_id=orion-vm-1234567890' +curl 'http://localhost:8080/vms/orion-vm-1234567890' ``` -**响应**(无 VM): +**列表响应**: ```json -{"status": "no_vm", "vm_id": null, "vm_ip": null} +{ + "status": "ok", + "count": 2, + "vms": [ + { + "status": "running", + "phase": "running", + "vm_id": "orion-vm-1234567890", + "domain": "orion.gitmega.com", + "target": "webhook", + "vm_ip": "192.168.221.87", + "uptime_secs": 60, + "log_file": "/var/log/orion-scheduler/orion-vm-1234567890-….log", + "error": null + } + ] +} ``` -### GET /logs/orion/stream +无过滤且空列表时仍为 `{ "status": "ok", "count": 0, "vms": [] }`。带 `?domain=` / `?vm_id=` 时返回单机对象(或 `phase: no_vm`)。 -SSE 流式端点,每 2 秒推送一次格式化日志。 +### GET /vms/{id} -**使用方式**: +按 `vm_id` 查单台;不存在时 **404** + `{ "phase": "no_vm", ... }`。 + +### GET /logs/orion/stream + +SSE;多 VM 时传 `?domain=` 或 `?vm_id=`。 ```bash -# 实时查看日志(终端持续刷新) -curl -N http://localhost:8080/logs/orion/stream +curl -N 'http://localhost:8080/logs/orion/stream?domain=orion.gitmega.com' ``` -**响应**:SSE 事件流,每 2 秒推送 journalctl 和 orion.log 的新增内容 - ### POST /shutdown -优雅关闭 VM 并退出服务。 - -**使用方式**: +必须指定 `?domain=` 或 `?vm_id=`;全关用 `POST /shutdown/all`。无参数 → **400**。 ```bash -curl -X POST http://localhost:8080/shutdown +curl -X POST 'http://localhost:8080/shutdown?domain=orion.gitmega.com' +curl -X POST http://localhost:8080/shutdown/all ``` **响应**: ```json -{"status": "ok", "message": "Shutdown initiated, VM will be stopped"} +{"status": "ok", "message": "VM for domain 'orion.gitmega.com' stopped", "domain": "orion.gitmega.com"} ``` ### GET /webhook @@ -126,7 +152,7 @@ Webhook 端点健康检查。 ### POST /webhook -接收来自 GitHub Actions 的更新请求。 +接收更新请求。默认异步:**202** + `status: provisioning`;后台 `handle_update`。`sync: true` 时阻塞至完成并返回 **200**。 **请求体**: @@ -137,6 +163,8 @@ Webhook 端点健康检查。 "scorpio_base_url": "https://git.gitmega.com", "scorpio_lfs_url": "https://git.gitmega.com", "target": "aws-gitmega", + "sync": false, + "replace": false, "image_path": "/path/to/image.qcow2", "image_digest": "sha256:abcd1234...", "image_disk_gb": 20, @@ -148,27 +176,39 @@ Webhook 端点健康检查。 | 字段 | 类型 | 必填 | 描述 | | --- | --- | --- | --- | | `action` | string | 否 | GitHub Actions 事件类型,仅记日志 | -| `server_ws` | string | 是 | Orion WebSocket URL,写入 VM 内 `.env` | +| `server_ws` | string | 是 | Orion WebSocket URL;**host 为 domain 唯一键** | | `scorpio_base_url` | string | 是 | Scorpio base URL,写入 `scorpio.toml` | | `scorpio_lfs_url` | string | 是 | Scorpio LFS URL,写入 `scorpio.toml` | -| `target` | string | 否 | 仅作日志标签(已废弃查表) | +| `target` | string | 否 | 仅作日志 / 展示标签(已废弃查表) | +| `sync` | bool | 否 | `true` 同步阻塞至部署完成(默认 `false` → 202) | +| `replace` | bool | 否 | 同 domain 已 Running 时强制重建(默认幂等 200) | | `image_path` | string | 否 | 本地 qcow2 镜像路径,与 `image_url` 互斥;未指定时使用 `default_image` | | `image_url` | string | 否 | 远程 HTTPS 镜像 URL,与 `image_path` 互斥 | -| `image_digest` | string | 否* | SHA256/SHA512 hash(`sha256:...` 或 `sha512:...`)。`image_path` 或 `image_url` 存在时必填 | +| `image_digest` | string | 否* | SHA256/SHA512 hash。`image_path` 或 `image_url` 存在时必填 | | `image_disk_gb` | u32 | 否 | VM 磁盘大小(GB),未指定时使用 `default_image` | | `image_cpus` | u32 | 否 | vCPU 数,未指定时使用 `default_image` | | `image_memory_mb` | u32 | 否 | 内存 MB,未指定时使用 `default_image` | -> **约束**:`image_path` 和 `image_url` 互斥,不能同时设置。提供了两者之一时 `image_digest` 必须提供。未传任何 `image_*` 字段时,scheduler 使用 `target_config.json` 中的 `default_image` 块(字段级 merge:payload 某字段为 `None` 时用 config 默认值)。 +> **约束**:`image_path` 和 `image_url` 互斥。提供了两者之一时 `image_digest` 必须提供。未传任何 `image_*` 时用 `default_image`(字段级 merge)。达 `max_vms` 且新 domain 时 **503**。 -**响应**: +**同 domain 响应**: + +| 情况 | HTTP | `status` | +|------|------|----------| +| 新建(异步) | 202 | `provisioning` | +| 新建(sync) | 200 | `ok` | +| 已 Running 且未 `replace` | 200 | `ok`(幂等,返回现有 `vm_id`) | +| 已 Provisioning | 409 | `conflict` | +| `max_vms` 已满(新 domain) | 503 | `error` | ```json { - "status": "ok", + "status": "provisioning", "vm_id": "orion-vm-1234567890", + "domain": "orion.gitmega.com", + "phase": "provisioning", "error": null, - "orion_log_file": "/var/log/orion-scheduler/orion-vm-1234567890-1746766200.log" + "orion_log_file": null } ``` @@ -188,6 +228,8 @@ pub enum VmPhase { pub struct VmInfo { pub id: String, + pub domain: String, // server_ws host;同 domain 唯一 + pub target: String, // 展示用标签 pub phase: VmPhase, pub ip: Option, pub created_at: std::time::Instant, @@ -195,50 +237,96 @@ pub struct VmInfo { pub error: Option, } +pub struct VmEntry { + pub info: VmInfo, + pub machine: Option, +} + pub struct AppState { - vm: Arc>>, - machine: Arc>>, // 持久化的 VM 连接 - pub config: SharedConfig, // 从 JSON 文件加载的配置(含 default_image) + /// key = domain(server_ws 主机名) + vms: Arc>>, + pub config: SharedConfig, // 含 default_image、可选 max_vms + /// 粗粒度 single-flight:创建 / shutdown 与信号关停共用,避免孤儿 qemu + update_lock: Arc>, } ``` -**Keep-alive 模式**:VM 在部署后保持运行状态,可通过 `GET /logs/orion/stream` 实时获取日志。 +**按域名多 VM**:不同 `server_ws` host 可同时 running;同 domain 冲突策略见 README。Keep-alive 下可通过 `GET /logs/orion/stream?domain=` 拉日志。 ### 4.2 生命周期 +整体分两段:**准入(HTTP 线程)** → **部署(`handle_update`,可后台)**。不同 domain 的 VM 独立并存;同 domain 最多一台。 + +#### 4.2.1 准入与冲突(`POST /webhook`) + +```mermaid +flowchart TD + A["POST /webhook"] --> B{"校验 server_ws / scorpio_*"} + B -->|失败| E400["400 error"] + B -->|通过| C["domain = host(server_ws)"] + C --> D{"该 domain 已有槽位?"} + + D -->|无| M{"max_vms 已满?"} + M -->|是| E503["503 max_vms"] + M -->|否| CREATE["分配 vm_id → 部署路径"] + + D -->|有| P{"phase?"} + P -->|Provisioning| E409["409 conflict
返回现有 vm_id"] + P -->|Running 且 replace=false| E200["200 幂等
返回现有 vm_id"] + P -->|Failed 或 Running+replace| CREATE +``` + +#### 4.2.2 同步 / 异步分流 + +```mermaid +flowchart TD + CREATE["准入通过,已有 vm_id"] --> S{"sync == true?"} + + S -->|否 默认| ASYNC["后台 spawn handle_update"] + ASYNC --> R202["立即 202
status=provisioning
domain + vm_id"] + R202 --> POLL["客户端轮询
GET /vms/id 或 /status?domain="] + POLL --> DONE_A["phase=running | failed"] + + S -->|是| SYNC["阻塞执行 handle_update"] + SYNC --> OK{"成功?"} + OK -->|是| R200["200 ok
phase=running"] + OK -->|否| R500["500 error
phase=failed"] ``` -[1] 接收 POST /webhook - ↓ -[1b] 立即返回 202 { vm_id, status: provisioning }(除非 sync: true) - ↓ -[2] 解析 webhook 内联 env URL(server_ws, scorpio_*)并 merge image 参数与 default_image - ↓ -[3] 检查现有 VM 并优雅关闭(如果存在) - ↓ -[4] 从 webhook 请求构造 ImageConfig,创建新 VM(keep-alive 模式) - ↓ -[5] 部署 Orion 文件到 VM - ↓ -[6] 替换环境变量(基于 webhook 内联 URL) - ↓ -[7] 启动 Orion 服务并获取日志 - ↓ -[8] 保存初始日志到文件 - ↓ -[9] 更新 VM 状态,VM 保持运行 - ↓ -[10] 返回成功响应 + +#### 4.2.3 单 domain 部署(`handle_update`) + +持 `update_lock`;**只**动当前 `domain`,其它 domain 的 VM 不受影响。 + +```mermaid +flowchart TD + H["handle_update(domain, vm_id, …)"] --> L["lock_update"] + L --> REG["set_vm_provisioning
写入 map[domain]"] + REG --> OLD{"同 domain 有旧 machine?"} + OLD -->|是| SD["shutdown 旧实例
(Failed / replace)"] + OLD -->|否| IMG + SD --> IMG["merge image_* + default_image
KeepAliveMachine::new"] + IMG --> DEP["SFTP 部署 orion + runner-config"] + DEP --> ENV["sed 写入 SERVER_WS / scorpio URLs"] + ENV --> START["systemctl start orion-runner"] + START --> LOG["落盘初始日志 → log_dir"] + LOG --> RUN["set_vm → phase=Running
VM keep-alive"] + RUN --> END["返回 vm_id"] + + DEP -.->|任一步失败| FAIL["set_vm_failed
phase=Failed"] + ENV -.-> FAIL + START -.-> FAIL + IMG -.-> FAIL ``` -**注意**:VM 在部署后保持运行状态,可通过 `GET /logs/orion/stream` 实时获取日志。 +完成后客户端可用 `GET /logs/orion/stream?domain=`(或 `?vm_id=`)拉实时日志;关停用 `POST /shutdown?domain=`,进程退出时关**全部**跟踪中的 VM。 ### 4.3 详细步骤 | 阶段 | 步骤 | 操作 | 说明 | | -------- | --- | ------------- | ---------------------------------------------------------------------------------- | -| **接收请求** | 1 | 接收 webhook | 解析必填 env URL(`server_ws`、`scorpio_base_url`、`scorpio_lfs_url`);merge 镜像参数与 `default_image` | -| **清理** | 2 | 清理旧 VM | 优雅关闭已有 VM(调用 `machine.shutdown()`) | -| **创建** | 3 | 构造 ImageConfig | 根据 webhook 镜像参数构造 `qlean::ImageConfig`(本地路径或远程 URL + digest);调用 `KeepAliveMachine::new()` 创建 VM | +| **接收请求** | 1 | 接收 webhook | 解析 `server_ws` → domain;冲突/幂等/`max_vms` 检查;异步则先 202 | +| **清理** | 2 | 清理同 domain 旧 VM | 仅对该 domain 调用 `shutdown`(其它 domain 不动) | +| **创建** | 3 | 构造 ImageConfig | 根据 webhook 镜像参数构造 `qlean::ImageConfig`;`KeepAliveMachine::new()` 创建 VM | | **部署** | 4 | 创建目录 | 在 VM 内创建 `/home/orion/orion-runner/` 目录 | | | 5 | 上传配置文件 | 通过 SFTP 上传 `run.sh`、`scorpio.toml`、`preflight.sh`、`cleanup.sh` | | | 6 | 上传 .env 文件 | 上传 `.env.prod` 重命名为 `.env` | @@ -272,6 +360,7 @@ pub struct AppState { "orion_source_dir": "/path/to/mega/orion", "orion_binary_path": "/path/to/mega/target/debug/orion", "ssh_public_key_path": "~/.ssh/orion_vm_access.pub", + "max_vms": 8, "default_image": { "image_path": "~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2", "image_digest": "sha256:753c28888c9d30fe4baef55c1d1dfa9a39431595eca940b7ad85d78d84f3d7a5", @@ -290,6 +379,7 @@ pub struct AppState { | `orion_source_dir` | string | 无默认值(必填) | Orion 源码目录(含 runner-config、systemd) | | `orion_binary_path` | string | 无默认值(必填) | Orion 二进制文件路径 | | `ssh_public_key_path` | string | 无默认值(必填) | SSH 公钥路径 | +| `max_vms` | u32 | 无(不限制) | 同时跟踪的 VM 上限(按 domain);新域超出 → 503 | | `default_image` | object | 见模板 | 默认 VM 镜像五参数;webhook 未传 `image_*` 时使用 | | `default_image.image_path` | string | — | 本地 qcow2 路径 | | `default_image.image_digest` | string | — | SHA256 校验和 | @@ -451,79 +541,56 @@ tracing::error!("[orion-deploy] Orion start failed: {}", error); #### 优雅关闭流程 -当服务收到 SIGTERM 或 SIGINT 信号时: +当服务收到 SIGTERM / SIGINT / SIGQUIT 时: ``` 1. 收到终止信号 -2. 停止接收新请求 -3. 检查是否有运行中的 VM -4. 如果有 VM: - a. 调用 machine.shutdown() 关闭 Orion 服务 - b. 调用 machine.stop() 停止 QEMU 进程 - c. 等待 VM 进程完全退出(最多 30 秒) - d. 如果超时,强制 kill QEMU 进程 -5. 清理状态文件(runs 目录下的临时文件) +2. 短超时获取 update_lock(失败仍继续) +3. take_all_machines():对每个已跟踪 VM 调用 machine.shutdown() +4. reap_qemu_from_runs():按 runs/*/qemu.pid 杀本数据树下残留 qemu + (不做宿主机全局 pkill -f qemu-system-x86,避免误伤其它 domain / 其它进程) +5. sweep_stale_runs():删除 qemu 已退出的 run 目录(overlay/seed 可至数 GB) 6. 退出进程 ``` -#### 实现机制 +HTTP `POST /shutdown?domain=` 只关一台;`POST /shutdown/all` 关全部跟踪 VM,服务继续监听。 -```rust -async fn graceful_shutdown(app_state: Arc) { - if let Some(mut vm_info) = app_state.vm.write().await.take() { - if let Some(machine) = vm_info.machine.take() { - machine.exec("systemctl stop orion-runner").await; - machine.shutdown().await; - } - } -} -``` +#### 实现要点 + +- 关停与 webhook 创建共用 `AppState::lock_update` / `try_lock_update`,避免在 `KeepAliveMachine::new` 与 `set_vm` 之间留下孤儿 qemu。 +- 启动时同样执行 `reap_qemu_from_runs` + `sweep_stale_runs`,清理上次 SIGKILL/崩溃留下的进程与磁盘。 #### 异常情况处理 | 场景 | 处理方式 | | ------- | -------------------------- | -| VM 关闭超时 | 强制 kill QEMU 进程(`kill -9`) | -| QEMU 僵死 | 使用 `fuser -k` 释放端口 | -| 残留进程 | 启动时检查并清理孤儿进程 | -| 文件锁 | 清理 `/var/lock/qemu/` 下的锁文件 | - -#### 启动时检查 - -服务启动时执行以下清理: - -```bash -# 清理残留 QEMU 进程 -pkill -9 qemu-system-x86 - -# 清理端口占用 -fuser -k 8080/tcp 2>/dev/null - -# 清理残留的 runs 目录 -rm -rf ~/.local/share/qlean/runs/* -``` +| VM 关闭超时 | KeepAlive / qlean 内部 stop;外加 run-dir pid 再杀 | +| 残留 qemu | 仅杀本 `XDG_DATA_HOME` 下 runs 登记的 pid | +| 文件锁 / 大 overlay | `sweep_stale_runs` 回收死目录 | +| HTTP 误关全部 | `/shutdown` 禁止无参数;需显式 `/shutdown/all` | ### 7.2 运行服务 ```bash # 构建 -cargo build --release +cargo build --release -p orion-scheduler -# 运行(需要 KVM 和 root 权限) -sudo env "PATH=$PATH" "RUSTUP_HOME=$RUSTUP_HOME" "CARGO_HOME=$CARGO_HOME" "HOME=$HOME" cargo run --release +# 运行(需要 KVM;工作目录能找到 target_config.json,或设 CONFIG_PATH) +CONFIG_PATH=./orion-scheduler/target_config.json \ + cargo run -p orion-scheduler -# 指定配置文件运行 -CONFIG_PATH=/path/to/target_config.json sudo env "PATH=$PATH" ... cargo run --release +# 可选:改监听地址 +LISTEN_ADDR=127.0.0.1:8081 cargo run -p orion-scheduler # 查看日志 -RUST_LOG=debug cargo run --release 2>&1 | grep -E '\[orion|webhook|vm' +RUST_LOG=debug cargo run -p orion-scheduler 2>&1 | grep -E '\[orion|webhook|vm|domain' ``` ## 8. 限制和未来工作 -- **状态持久化**:VM 状态持久化在内存中,服务重启后 VM 状态丢失 -- **安全**:没有 webhook 签名验证 -- **错误处理**:需要更健壮的错误恢复 -- **并发请求**:不支持 - 一次只能有一个 VM -- **日志持久化**:初始日志持久化到文件,实时日志从 journalctl 读取 -- **Orion 二进制分发**:通过 `target_config.json` 的 `orion_binary_path` 配置本地路径,未来改为通过 GitHub Actions 上传到 GitHub Releases,VM 直接从 Releases 下载,支持多架构和多版本管理 +- **状态持久化**:VM 状态仅在内存;进程重启后 map 清空(磁盘上的 qemu 靠启动 reap) +- **安全**:没有 webhook 签名验证(生产建议经 mono 内网 + Bearer) +- **并发锁**:`update_lock` 目前全局;不同 domain 创建仍串行,后续可改 per-domain Mutex +- **多 VM**:已按 domain 唯一并存;可选 `max_vms`;不默认支持同机多 scheduler 进程(可用 `LISTEN_ADDR` + `XDG_DATA_HOME` 运维隔离) +- **日志持久化**:初始日志落盘;SSE 从 journalctl / orion.log 读 +- **Orion 二进制分发**:仍靠本地 `orion_binary_path`;未来 S3 / Releases 拉取见 ARTIFACT.md diff --git a/orion-scheduler/README.md b/orion-scheduler/README.md index 77b3f3719..1cb09b4b5 100644 --- a/orion-scheduler/README.md +++ b/orion-scheduler/README.md @@ -4,13 +4,14 @@ Orion Scheduler 是一个常驻运行的服务,负责接收来自 GitHub Actio ```mermaid flowchart LR - GHA["GitHub Actions"] -->|"POST /webhook"| Sched["orion-scheduler
(axum HTTP server)"] - Sched -->|"qlean"| VM["microVM
(KVM, custom image)"] - Sched -->|"SFTP"| VM - Sched -->|"SSH exec"| VM - User["开发者 / 监控"] -->|"GET /status
/logs/orion/*
/scorpio/*"| Sched + Mono["mono / GHA"] -->|"POST /webhook"| Sched["orion-scheduler"] + Sched -->|"qlean"| VMa["microVM domain A"] + Sched -->|"qlean"| VMb["microVM domain B"] + User["开发者 / 监控"] -->|"GET /status
GET /vms/{id}"| Sched ``` +单进程可同时托管**多台 VM**,唯一性由 `server_ws` 的**主机名(domain)**决定:同一 domain 最多一台;不同 domain 互不踢掉。 + --- ## 快速开始 @@ -35,6 +36,7 @@ flowchart LR "orion_source_dir": "/home/user/mega/orion", "orion_binary_path": "/home/user/mega/target/debug/orion", "ssh_public_key_path": "~/.ssh/orion_vm_access.pub", + "max_vms": 8, "default_image": { "image_path": "~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2", "image_digest": "sha256:753c28888c9d30fe4baef55c1d1dfa9a39431595eca940b7ad85d78d84f3d7a5", @@ -45,6 +47,8 @@ flowchart LR } ``` +可选 `max_vms`:并发 VM 上限(按 domain 计数);省略则不限制。 + 配置文件路径可通过 `CONFIG_PATH` 环境变量指定(默认:`target_config.json`)。 --- @@ -61,8 +65,8 @@ sequenceDiagram participant VM as microVM CI->>SC: POST /webhook { server_ws, scorpio_* } - SC->>SC: merge image params with default_image - SC->>Q: 已有运行中 VM? 先关闭 + SC->>SC: domain = host(server_ws); 冲突/幂等检查 + SC->>Q: 仅关闭同 domain 旧 VM(若 replace/Failed) SC->>Q: KeepAliveMachine::new(custom_image) Q->>VM: 启动 QEMU + cloud-init SC->>VM: 注入 SSH 公钥 (orion_vm_access.pub) @@ -71,13 +75,21 @@ sequenceDiagram SC->>VM: systemctl start orion-runner VM-->>SC: journalctl 初始日志 SC->>SC: 写入 log_dir - SC-->>CI: 202 { vm_id, status: provisioning } - Note over SC,VM: VM 保持运行
支持后续日志查询 - CI->>SC: GET /status (poll) - SC-->>CI: { phase: running, vm_ip, ... } + SC-->>CI: 202 { vm_id, domain, status: provisioning } + Note over SC,VM: 该 domain 的 VM 保持运行;其他 domain 不受影响 + CI->>SC: GET /vms/{vm_id} 或 GET /status?domain= + SC-->>CI: { phase: running, domain, vm_ip, ... } ``` -默认 **异步** 模式:`POST /webhook` 立即返回 `202 Accepted` 与 `vm_id`,后台执行部署;客户端轮询 `GET /status` 直至 `phase=running`。 +默认 **异步** 模式:`POST /webhook` 立即返回 `202 Accepted` 与 `vm_id` + `domain`,后台执行部署;客户端轮询 `GET /vms/{id}` 或 `GET /status?domain=` 直至 `phase=running`。 + +**同 domain 再次启动**: + +| 当前状态 | 行为 | +|----------|------| +| 无 / Failed | 允许新建(Failed 覆盖) | +| Provisioning | `409 Conflict`,返回现有 `vm_id` | +| Running | 幂等 `200`,返回现有 `vm_id`;`replace: true` 时强制重建 | GHA 等需同步等待的调用方可传 `"sync": true` 保留旧行为(阻塞至完成,返回 `200`)。 @@ -95,12 +107,14 @@ Admin 用户通过 Mega UI Orion Client 页调用 mono `POST /api/v1/orion/runne | ---- | -------------------- | ------------------------------------------------- | ----------------------------------------------- | | GET | `/health` | 服务健康检查 | `{ "status": "healthy", ... }` | | GET | `/webhook` | Webhook 端点连通性检查 | `{ "status": "ok", "vm_id": null, ... }` | -| POST | `/webhook` | 触发部署,详见下方参数说明 | 默认 `202 { "status": "provisioning", "vm_id" }`;`sync: true` 时 `200 { "status": "ok", "vm_id", "orion_log_file" }` | -| GET | `/status` | 当前虚拟机状态 | `{ "phase": "provisioning"|"running"|"failed"|"no_vm", "vm_id", "vm_ip", "error", ... }` | -| GET | `/logs/orion/stream` | SSE 流式推送,每 2 秒推送新增日志 | `text/event-stream` | -| GET | `/scorpio/status` | Scorpio FUSE 挂载点、目录、进程状态 | JSON | -| GET | `/scorpio/config` | 直接读取 VM 内 `/home/orion/orion-runner/scorpio.toml` | `{ "path", "content" }` | -| POST | `/shutdown` | 仅关闭虚拟机,服务进程保持运行 | `{ "status": "ok", "message" }` | +| POST | `/webhook` | 触发部署(按 `server_ws` 域名唯一) | 默认 `202 { provisioning, vm_id, domain }`;Running 幂等 `200`;Provisioning `409` | +| GET | `/status` | 所有 VM 列表;可选 `?domain=` / `?vm_id=` | `{ "count", "vms": [...] }` 或单机摘要 | +| GET | `/vms/{id}` | 按 vm_id 查询单台 | `{ phase, domain, vm_ip, ... }` / 404 | +| GET | `/logs/orion/stream` | SSE 日志;需 `?domain=` 或 `?vm_id=`(多 VM 时) | `text/event-stream` | +| GET | `/scorpio/status` | Scorpio 状态;可选 `?domain=` / `?vm_id=` | JSON | +| GET | `/scorpio/config` | 读取 scorpio.toml;可选选择器 | `{ "path", "content" }` | +| POST | `/shutdown` | **必须** `?domain=` 或 `?vm_id=` | `{ "status": "ok", "domain" }` | +| POST | `/shutdown/all` | 关闭全部跟踪中的 VM | `{ "status": "ok" }` | ### POST /webhook 请求体 @@ -122,6 +136,7 @@ curl -X POST http://localhost:8080/webhook \ | `target` | string | 否 | 仅作日志标签(已废弃查表,GHA 可逐步移除) | | `action` | string | 否 | GitHub Actions 事件类型,仅作日志记录 | | `sync` | bool | 否 | 为 `true` 时同步阻塞至部署完成(默认 `false`,立即 202 返回) | +| `replace` | bool | 否 | Running 状态下为 `true` 时强制重建(默认幂等返回已有实例) | | `image_path` | string | 否 | 本地 qcow2 镜像路径;未指定时使用 `default_image.image_path` | | `image_url` | string | 否 | 远程 HTTPS URL | | `image_digest` | string | 条件必填 | 镜像 SHA256/SHA512 校验和,提供 `image_path` 或 `image_url` 时必须指定 | @@ -221,6 +236,7 @@ sudo bash scripts/build-custom-image.sh | `orion_source_dir` | string | 无默认值(必填) | Orion 源码目录(含 runner-config、systemd) | | `orion_binary_path` | string | 无默认值(必填) | Orion 二进制文件路径 | | `ssh_public_key_path` | string | 无默认值(必填) | SSH 公钥路径 | +| `max_vms` | u32 | 无(不限制) | 同时跟踪的 domain/VM 上限;超出新 domain 返回 503 | | `default_image` | object | 见模板 | 默认 VM 镜像参数;webhook 未传 `image_*` 时使用 | ### `default_image` @@ -244,12 +260,12 @@ orion-scheduler/ ├── src/ │ ├── main.rs # axum 入口 + 信号处理 │ ├── handlers.rs # HTTP 端点 + 日志格式化 -│ ├── state.rs # AppState:VM info + KeepAliveMachine -│ ├── config.rs # 读取/解析 target_config.json +│ ├── state.rs # AppState:按 domain 的 HashMap + KeepAliveMachine +│ ├── config.rs # 读取/解析 target_config.json(含 max_vms) │ ├── keep_alive.rs # qlean::Machine 持久化包装 -│ ├── orion_deployer.rs # handle_update 编排(webhook 主流程) +│ ├── orion_deployer.rs # handle_update 编排(按 domain 隔离) │ ├── vm_manager.rs # SFTP 上传、sed 环境变量替换、systemctl 启停 -│ └── vm_cleanup.rs # qlean runs 目录泄漏清理 +│ └── vm_cleanup.rs # 按 runs/qemu.pid 回收(无全局 pkill) ├── scripts/ │ └── build-custom-image.sh # chroot 离线预装 + qcow2 压缩 + 发布 ├── .github/workflows/ @@ -267,25 +283,29 @@ orion-scheduler/ | 信号 / 动作 | VM | 服务进程 | 说明 | | -------------------------- | --- | -------- | ---------------------- | -| `Ctrl+C` / SIGINT | 关闭 | 退出 | 优雅关闭 | -| SIGTERM | 关闭 | 退出 | 同上 | -| SIGQUIT | 关闭 | 退出 | 同上 | -| `POST /shutdown` | 关闭 | **保持运行** | 仅回收虚拟机 | -| `pkill -9 orion-scheduler` | 残留 | 强制终止 | 不优雅关闭,虚拟机将变为孤儿进程,需手动清理 | +| `Ctrl+C` / SIGINT | **全部**关闭 | 退出 | 关停所有跟踪中的 domain VM,再退出 | +| SIGTERM / SIGQUIT | **全部**关闭 | 退出 | 同上 | +| `POST /shutdown?domain=` | **一台**关闭 | **保持运行** | 必须带 `domain` 或 `vm_id` | +| `POST /shutdown/all` | **全部**关闭 | **保持运行** | 运维用 | +| `pkill -9 orion-scheduler` | 可能残留 | 强制终止 | 不优雅关闭;孤儿 qemu 需手动清理或下次启动 `reap_qemu_from_runs` | | 环境变量 | 默认 | 说明 | | ------------- | -------------------- | ----------------------- | -| `CONFIG_PATH` | `target_config.json` | 配置文件路径 | +| `CONFIG_PATH` | 自动探测 / `target_config.json` | 配置文件路径;见 `config::default_config_path` | +| `LISTEN_ADDR` | `0.0.0.0:8080` | HTTP 监听地址(同机多进程时可覆盖) | | `RUST_LOG` | `info` | tracing 日志级别,常用 `debug` | +| `XDG_DATA_HOME` | `~/.local/share` | qlean `runs/` / `images/` 根目录;多进程隔离时可分目录 | --- ## 调试与 SSH 进入 VM -部署完成后从 `/status` 获取 `vm_ip`,使用构建脚本预装的 SSH 密钥登录: +部署完成后从 `GET /status` 或 `GET /vms/{id}` 获取 `vm_ip`,使用构建脚本预装的 SSH 密钥登录: ```bash -ssh -i ~/.ssh/orion_vm_access root@ +VM_IP=$(curl -s 'http://localhost:8080/status?domain=orion.gitmega.com' | jq -r .vm_ip) +# 或:curl -s http://localhost:8080/status | jq -r '.vms[0].vm_ip' +ssh -i ~/.ssh/orion_vm_access root@$VM_IP ``` 完整调试流程参见 `[TESTING.md](TESTING.md)`。 @@ -368,7 +388,7 @@ sudo journalctl -u orion-scheduler -f | 回滚 | 下载旧版本 tarball → 同上 | | 检查当前版本 | `cat /opt/orion-scheduler/bin/orion-scheduler --version`(若启用了 clap version)或 `cat ~/.../VERSION`(解压后) | | 停服并保留 VM | `sudo systemctl stop orion-scheduler`;VM 不会被关闭 | -| 完全停服 | 先 `POST /shutdown` 关闭 VM,再 `systemctl stop` | +| 完全停服 | 先 `POST /shutdown/all`(或逐台 `?domain=`)关闭 VM,再 `systemctl stop` | > **注意**:必须从 Linux x86_64 host 上跑。orion-scheduler 通过 `qlean` 依赖 `kvm-ioctls`,不可在 macOS / ARM64 上运行。 diff --git a/orion-scheduler/TESTING.md b/orion-scheduler/TESTING.md index 5f7b95b48..9c61436b6 100644 --- a/orion-scheduler/TESTING.md +++ b/orion-scheduler/TESTING.md @@ -1,6 +1,6 @@ # 测试方法 -本地调试、API 测试、服务管理和常见问题排查。 +本地调试、API 测试、服务管理和常见问题排查。与当前实现一致:**webhook 必须内联 `server_ws` / scorpio URL**;多 VM 按 domain 唯一。 ## 前提条件 @@ -10,7 +10,7 @@ ssh-keygen -t ed25519 -f ~/.ssh/orion_vm_access -N "" -C "orion-scheduler" # 配置文件 cp orion-scheduler/target_config.json.template orion-scheduler/target_config.json -# 编辑 target_config.json,填入本机路径 +# 编辑 target_config.json,填入本机路径;可选 max_vms ``` 自定义 VM 镜像的构建与上传见 [§4 构建镜像并上传到 S3](#4-构建镜像并上传到-s3)。 @@ -20,29 +20,40 @@ cp orion-scheduler/target_config.json.template orion-scheduler/target_config.jso ## 1. 快速开始 ```bash -# 构建并启动 scheduler(需 root/KVM) +# 构建并启动 scheduler(需 KVM;CONFIG_PATH 指向你的配置) cargo build -p orion-scheduler -sudo env "PATH=$PATH" "RUSTUP_HOME=$RUSTUP_HOME" "CARGO_HOME=$CARGO_HOME" "HOME=$HOME" \ - cargo run -p orion-scheduler +CONFIG_PATH=./orion-scheduler/target_config.json cargo run -p orion-scheduler -# 触发 VM + Orion worker(推荐:本地 debian-13-buck2 镜像) -curl -X POST http://localhost:8080/webhook \ +# 触发 VM(必填三个 URL;domain = server_ws 的 host) +curl -i -X POST http://localhost:8080/webhook \ -H "Content-Type: application/json" \ -d '{ - "target": "k3s-buck2hub", + "server_ws": "wss://orion.gitmega.com/ws", + "scorpio_base_url": "https://git.gitmega.com", + "scorpio_lfs_url": "https://git.gitmega.com", "image_path": "~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2", "image_digest": "sha256:753c28888c9d30fe4baef55c1d1dfa9a39431595eca940b7ad85d78d84f3d7a5", "image_disk_gb": 30, "image_cpus": 8, "image_memory_mb": 16000 }' +# 期望:HTTP 202,body 含 vm_id、domain、status=provisioning -# VM 与日志 -curl http://localhost:8080/status -curl -N http://localhost:8080/logs/orion/stream +# 列表 / 单机 / 日志 +curl -s http://localhost:8080/status | jq . +curl -s 'http://localhost:8080/status?domain=orion.gitmega.com' | jq . +curl -N 'http://localhost:8080/logs/orion/stream?domain=orion.gitmega.com' ``` -`image_path` 支持 `~/...` 或绝对路径。其他 webhook 变体(默认镜像、远程 `image_url`)见 [§2 Webhook](#webhook)。 +`image_path` 支持 `~/...` 或绝对路径。未传任何 `image_*` 时使用配置里的 `default_image`。 + +### 单元测试(不启 VM) + +```bash +cargo test -p orion-scheduler --bins +``` + +覆盖 domain 解析、两 domain 共存、`max_vms`/`merge` 等逻辑(见 `state` / `handlers` / `orion_deployer` 测试模块)。 --- @@ -58,66 +69,123 @@ curl http://localhost:8080/health ### Webhook ```bash -# GET +# GET — 连通性 curl http://localhost:8080/webhook -# POST — 默认 Debian 镜像 -curl -X POST http://localhost:8080/webhook \ +# POST — 仅用 default_image(仍须三个 URL) +curl -i -X POST http://localhost:8080/webhook \ + -H "Content-Type: application/json" \ + -d '{ + "server_ws": "wss://orion.gitmega.com/ws", + "scorpio_base_url": "https://git.gitmega.com", + "scorpio_lfs_url": "https://git.gitmega.com" + }' + +# POST — 第二个 domain(应与第一台并存) +curl -i -X POST http://localhost:8080/webhook \ -H "Content-Type: application/json" \ - -d '{"target": "aws-gitmega"}' + -d '{ + "server_ws": "wss://orion.xuanwu.openatom.cn/ws", + "scorpio_base_url": "https://git.xuanwu.openatom.cn", + "scorpio_lfs_url": "https://git.xuanwu.openatom.cn" + }' + +# POST — 同 domain 再启(Running → 幂等 200;Provisioning → 409) +curl -i -X POST http://localhost:8080/webhook \ + -H "Content-Type: application/json" \ + -d '{ + "server_ws": "wss://orion.gitmega.com/ws", + "scorpio_base_url": "https://git.gitmega.com", + "scorpio_lfs_url": "https://git.gitmega.com" + }' -# POST — 本地自定义镜像(字段同 §1 快速开始) -curl -X POST http://localhost:8080/webhook \ +# POST — 强制重建同 domain +curl -i -X POST http://localhost:8080/webhook \ -H "Content-Type: application/json" \ - -d '{"target": "gcp-buck2hub", "image_path": "~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2", "image_digest": "sha256:...", "image_disk_gb": 20, "image_cpus": 4, "image_memory_mb": 8192}' + -d '{ + "server_ws": "wss://orion.gitmega.com/ws", + "scorpio_base_url": "https://git.gitmega.com", + "scorpio_lfs_url": "https://git.gitmega.com", + "replace": true + }' -# POST — 远程镜像(image_url + image_digest + 可选 disk/cpus/memory) -curl -X POST http://localhost:8080/webhook \ +# POST — 同步阻塞至完成(GHA 可用) +curl -i -X POST http://localhost:8080/webhook \ -H "Content-Type: application/json" \ - -d '{"target": "aws-gitmega", "image_url": "https://...", "image_digest": "sha256:...", "image_disk_gb": 20, "image_cpus": 4, "image_memory_mb": 8192}' + -d '{ + "server_ws": "wss://orion.gitmega.com/ws", + "scorpio_base_url": "https://git.gitmega.com", + "scorpio_lfs_url": "https://git.gitmega.com", + "sync": true + }' ``` +> 旧版仅 `{"target":"..."}` 查 `targets` 表已**不再支持**。 + ### VM 状态 ```bash -curl http://localhost:8080/status -# {"status": "running", "vm_id": "orion-vm-xxx", "vm_ip": "192.168.221.x", "uptime_secs": 60, ...} +# 全部 +curl -s http://localhost:8080/status | jq . +# {"status":"ok","count":2,"vms":[{phase,vm_id,domain,vm_ip,...},...]} + +# 按 domain / vm_id +curl -s 'http://localhost:8080/status?domain=orion.gitmega.com' | jq . +curl -s http://localhost:8080/vms/ | jq . +``` + +异步部署时轮询直至 `phase` 为 `running` 或 `failed`: + +```bash +VM_ID=... # 从 202 响应取出 +while true; do + curl -s "http://localhost:8080/vms/$VM_ID" | jq -c '{phase,vm_ip,error}' + sleep 5 +done ``` ### SSH 进入 VM ```bash -VM_IP=$(curl -s http://localhost:8080/status | jq -r .vm_ip) +VM_IP=$(curl -s 'http://localhost:8080/status?domain=orion.gitmega.com' | jq -r .vm_ip) ssh -i ~/.ssh/orion_vm_access root@$VM_IP ``` -部署时 scheduler 会在 guest 内创建 **8 GB** swap 文件(`/swapfile`,写入 `/etc/fstab`),减轻全量 buck2 编译时的内存压力。可用 `swapon --show` / `free -h` 确认。 +部署时 scheduler 会在 guest 内创建 **8 GB** swap 文件(`/swapfile`,写入 `/etc/fstab`)。可用 `swapon --show` / `free -h` 确认。 ### 日志 | 端点 | 格式 | 说明 | |------|------|------| -| `GET /logs/orion/stream` | SSE | 每 2 秒推送;`curl -N` 持续监控 | +| `GET /logs/orion/stream?domain=` 或 `?vm_id=` | SSE | 多 VM 时**建议**带选择器;`curl -N` | ```bash -curl -N http://localhost:8080/logs/orion/stream +curl -N 'http://localhost:8080/logs/orion/stream?domain=orion.gitmega.com' ``` -服务端调试日志:`RUST_LOG=debug cargo run -p orion-scheduler`;systemd 部署用 `journalctl -u orion-scheduler -f`。 +服务端调试:`RUST_LOG=debug cargo run -p orion-scheduler`;systemd:`journalctl -u orion-scheduler -f`。 -### Scorpio 状态 +### Scorpio ```bash -curl http://localhost:8080/scorpio/status +curl -s 'http://localhost:8080/scorpio/status?domain=orion.gitmega.com' | jq . +curl -s 'http://localhost:8080/scorpio/config?domain=orion.gitmega.com' | jq . ``` ### 关闭 ```bash -curl -X POST http://localhost:8080/shutdown -# 仅停 VM,scheduler 继续运行 +# 关一台(必须带参数) +curl -X POST 'http://localhost:8080/shutdown?domain=orion.gitmega.com' +# 或 +curl -X POST 'http://localhost:8080/shutdown?vm_id=orion-vm-xxx' + +# 关全部跟踪 VM(scheduler 继续跑) +curl -X POST http://localhost:8080/shutdown/all ``` +无 `domain`/`vm_id` 的 `POST /shutdown` → **400**。 + --- ## 3. 服务管理 @@ -125,12 +193,14 @@ curl -X POST http://localhost:8080/shutdown ### 停止与检查 ```bash -# 优雅:先关 VM(见上),再停 scheduler +# 优雅:先关 VM,再停 scheduler +curl -X POST http://localhost:8080/shutdown/all kill -TERM -# 强制(不关闭 VM) +# 强制(可能残留 qemu) pkill -9 -f orion-scheduler -sudo pkill -9 -f qemu-system-x86 # 清理残留 QEMU +# 勿再全局 pkill 所有 qemu-system-x86(会误伤其它 domain / 其它用户) +# 下次启动会 reap 本用户 XDG_DATA_HOME 下 runs/*/qemu.pid ps aux | grep -E "orion-scheduler|qemu-system" | grep -v grep fuser 8080/tcp 2>/dev/null || echo "Port 8080 is free" @@ -140,10 +210,13 @@ fuser 8080/tcp 2>/dev/null || echo "Port 8080 is free" | 操作 | VM | scheduler | 说明 | |------|-----|-----------|------| -| `Ctrl+C` / SIGTERM / SIGQUIT | 停止 | 停止 | 先关 VM 再退出 | -| `POST /shutdown` | 停止 | **继续** | 仅关 VM,适合换 CL 重跑 | +| `Ctrl+C` / SIGTERM / SIGQUIT | **全部**停止 | 停止 | `take_all_machines` + run-dir reap | +| `POST /shutdown?domain=` | **一台**停止 | **继续** | | +| `POST /shutdown/all` | **全部**停止 | **继续** | | | `pkill -9 -f orion-scheduler` | 可能残留 | 停止 | 不优雅 | +`LISTEN_ADDR` 可改端口;`XDG_DATA_HOME` 可隔离 qlean 数据目录。 + --- ## 4. 构建镜像并上传到 S3 @@ -167,10 +240,14 @@ aws s3 cp ~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2 \ | 问题 | 排查 | |------|------| -| KVM 权限错误 | `/dev/kvm` 权限;用户是否在 `kvm` 组 | +| webhook 400 / missing URL | 必须带 `server_ws`、`scorpio_base_url`、`scorpio_lfs_url` | +| 409 conflict | 同 domain 正在 provisioning;等完成或查 `/status?domain=` | +| 幂等 200 | 同 domain 已 Running;要重建加 `"replace": true` | +| 503 max_vms | 提高配置 `max_vms` 或先 `shutdown` 腾出 slot | +| KVM 权限错误 | `/dev/kvm`;用户是否在 `kvm` 组 | | QEMU 桥接失败 | `/etc/qemu/bridge.conf` 是否 `allow qlbr0` | | VM 启动超时 | cloud-init、SSH 是否可达 | -| Orion 启动失败 | `curl -N http://localhost:8080/logs/orion/stream` | -| Scorpio 挂载问题 | `curl http://localhost:8080/scorpio/status` | -| 状态仍 running 但 VM 已死 | 重启 scheduler 或查 QEMU 进程 | +| Orion 启动失败 | `curl -N '.../logs/orion/stream?domain=...'` | +| Scorpio 挂载问题 | `curl '.../scorpio/status?domain=...'` | +| 重启后状态丢了 | 内存 map;磁盘 qemu 靠启动 reap;重新 POST webhook | | 进 VM 调试 | [SSH 进入 VM](#ssh-进入-vm) | diff --git a/orion-scheduler/src/config.rs b/orion-scheduler/src/config.rs index a5a7e4388..0768751e9 100644 --- a/orion-scheduler/src/config.rs +++ b/orion-scheduler/src/config.rs @@ -47,6 +47,8 @@ pub struct Config { orion_binary_path: String, ssh_public_key_path: String, default_image: DefaultImageConfig, + /// Max concurrent VMs (by domain). `None` = unlimited. + max_vms: Option, } /// Expand a leading `~` or `~/` to `$HOME`. Other paths are returned unchanged. @@ -80,6 +82,7 @@ impl Config { orion_binary_path, ssh_public_key_path, default_image, + max_vms: None, } } @@ -123,6 +126,7 @@ impl Config { orion_binary_path, ssh_public_key_path, default_image: parsed.default_image.unwrap_or_default(), + max_vms: parsed.max_vms, }) } @@ -145,6 +149,10 @@ impl Config { pub fn default_image(&self) -> &DefaultImageConfig { &self.default_image } + + pub fn max_vms(&self) -> Option { + self.max_vms + } } pub fn default_config_path() -> Option { @@ -195,6 +203,8 @@ struct ConfigFile { ssh_public_key_path: Option, #[serde(default)] default_image: Option, + #[serde(default)] + max_vms: Option, } pub type SharedConfig = Arc>; diff --git a/orion-scheduler/src/handlers.rs b/orion-scheduler/src/handlers.rs index 0195293c1..dd3143185 100644 --- a/orion-scheduler/src/handlers.rs +++ b/orion-scheduler/src/handlers.rs @@ -4,7 +4,7 @@ use std::{ }; use axum::{ - extract::State, + extract::{Path, Query, State}, http::StatusCode, response::{ IntoResponse, Json, @@ -17,7 +17,7 @@ use tokio::time::interval; use crate::{ config::{DefaultImageConfig, TargetConfig}, orion_deployer, - state::AppState, + state::{AppState, VmPhase}, vm_cleanup, }; @@ -36,6 +36,10 @@ pub struct ImageParams { pub struct WebhookResponse { pub status: String, pub vm_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub domain: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub phase: Option, pub error: Option, /// Path to the log file (not the contents) pub orion_log_file: Option, @@ -50,6 +54,9 @@ pub struct GithubWebhookPayload { /// When true, block until VM provisioning completes (legacy GHA behavior). #[serde(default)] pub sync: bool, + /// Force recreate when a Running VM exists for the same domain. + #[serde(default)] + pub replace: bool, pub server_ws: String, pub scorpio_base_url: String, pub scorpio_lfs_url: String, @@ -118,6 +125,7 @@ mod merge_tests { action: None, target: None, sync: false, + replace: false, server_ws: "ws://orion.test/ws".into(), scorpio_base_url: "http://git.test".into(), scorpio_lfs_url: "http://git.test".into(), @@ -144,6 +152,7 @@ mod merge_tests { action: None, target: None, sync: false, + replace: false, server_ws: "ws://orion.test/ws".into(), scorpio_base_url: "http://git.test".into(), scorpio_lfs_url: "http://git.test".into(), @@ -165,24 +174,45 @@ pub async fn webhook_get_handler() -> Json { Json(WebhookResponse { status: "ok".to_string(), vm_id: None, + domain: None, + phase: None, error: None, orion_log_file: None, }) } -/// POST /webhook - receives update requests from GitHub Actions +fn vm_json(vm: &crate::state::VmInfo) -> serde_json::Value { + let phase = vm.phase.as_str(); + let uptime_secs = if vm.phase == VmPhase::Running { + Some(vm.created_at.elapsed().as_secs()) + } else { + None + }; + serde_json::json!({ + "status": phase, + "phase": phase, + "vm_id": vm.id, + "domain": vm.domain, + "target": vm.target, + "vm_ip": vm.ip, + "uptime_secs": uptime_secs, + "log_file": vm.log_file, + "error": vm.error + }) +} + +/// POST /webhook - receives update requests (async by default; one VM per server_ws domain). pub async fn webhook_post_handler( State(state): State>, Json(payload): Json, ) -> impl IntoResponse { tracing::info!( - "Received webhook: action={:?}, target={:?}, sync={}, server_ws={}, scorpio_base_url={}, scorpio_lfs_url={}", + "Received webhook: action={:?}, target={:?}, sync={}, replace={}, server_ws={}", payload.action, payload.target, payload.sync, + payload.replace, payload.server_ws, - payload.scorpio_base_url, - payload.scorpio_lfs_url ); if let Err(e) = orion_deployer::validate_runner_env( @@ -191,13 +221,92 @@ pub async fn webhook_post_handler( &payload.scorpio_lfs_url, ) { tracing::error!("Invalid runner env: {:?}", e); - let response = WebhookResponse { - status: "error".to_string(), - vm_id: None, - error: Some(e.to_string()), - orion_log_file: None, - }; - return (StatusCode::BAD_REQUEST, Json(response)).into_response(); + return ( + StatusCode::BAD_REQUEST, + Json(WebhookResponse { + status: "error".to_string(), + vm_id: None, + domain: None, + phase: None, + error: Some(e.to_string()), + orion_log_file: None, + }), + ) + .into_response(); + } + + let domain = match orion_deployer::domain_from_server_ws(&payload.server_ws) { + Ok(d) => d, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + Json(WebhookResponse { + status: "error".to_string(), + vm_id: None, + domain: None, + phase: None, + error: Some(e.to_string()), + orion_log_file: None, + }), + ) + .into_response(); + } + }; + + // Conflict / idempotency checks (hold update lock briefly). + { + let _guard = state.lock_update().await; + if let Some(existing) = state.get_vm_by_domain(&domain).await { + match existing.phase { + VmPhase::Provisioning => { + return ( + StatusCode::CONFLICT, + Json(WebhookResponse { + status: "conflict".to_string(), + vm_id: Some(existing.id.clone()), + domain: Some(domain), + phase: Some(existing.phase.as_str().to_string()), + error: Some("VM already provisioning for this domain".to_string()), + orion_log_file: existing.log_file.clone(), + }), + ) + .into_response(); + } + VmPhase::Running if !payload.replace => { + return ( + StatusCode::OK, + Json(WebhookResponse { + status: "ok".to_string(), + vm_id: Some(existing.id.clone()), + domain: Some(domain), + phase: Some(existing.phase.as_str().to_string()), + error: None, + orion_log_file: existing.log_file.clone(), + }), + ) + .into_response(); + } + VmPhase::Running | VmPhase::Failed => { + // replace=true or Failed: allow recreate (handle_update will shut down). + } + } + } else if let Some(max) = state.config.read().await.max_vms() { + let count = state.vm_count().await; + if count >= max { + return ( + StatusCode::SERVICE_UNAVAILABLE, + Json(WebhookResponse { + status: "error".to_string(), + vm_id: None, + domain: Some(domain), + phase: None, + error: Some(format!("max_vms limit reached ({max})")), + orion_log_file: None, + }), + ) + .into_response(); + } + } } let default_image = state.config.read().await.default_image().clone(); @@ -214,6 +323,7 @@ pub async fn webhook_post_handler( .target .clone() .unwrap_or_else(|| "webhook".to_string()); + let domain_for_task = domain.clone(); if payload.sync { let state_clone = state.clone(); @@ -222,6 +332,7 @@ pub async fn webhook_post_handler( let rt = tokio::runtime::Handle::current(); rt.block_on(orion_deployer::handle_update( &state_clone, + &domain_for_task, &label, &vm_id_clone, target_config, @@ -233,46 +344,62 @@ pub async fn webhook_post_handler( return match result { Ok(Ok(_vm_id)) => { tracing::info!("Successfully created VM: {}", _vm_id); - let orion_log_file = state.get_vm().await.and_then(|vm| vm.log_file); - let response = WebhookResponse { - status: "ok".to_string(), - vm_id: Some(_vm_id), - error: None, - orion_log_file, - }; - (StatusCode::OK, Json(response)).into_response() + let orion_log_file = state.get_vm_by_id(&_vm_id).await.and_then(|vm| vm.log_file); + ( + StatusCode::OK, + Json(WebhookResponse { + status: "ok".to_string(), + vm_id: Some(_vm_id), + domain: Some(domain), + phase: Some("running".to_string()), + error: None, + orion_log_file, + }), + ) + .into_response() } Ok(Err(e)) => { tracing::error!("Failed to handle update: {:?}", e); - let response = WebhookResponse { - status: "error".to_string(), - vm_id: Some(vm_id), - error: Some(e.to_string()), - orion_log_file: None, - }; - (StatusCode::INTERNAL_SERVER_ERROR, Json(response)).into_response() + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(WebhookResponse { + status: "error".to_string(), + vm_id: Some(vm_id), + domain: Some(domain), + phase: Some("failed".to_string()), + error: Some(e.to_string()), + orion_log_file: None, + }), + ) + .into_response() } Err(e) => { tracing::error!("Task join error: {:?}", e); - let response = WebhookResponse { - status: "error".to_string(), - vm_id: None, - error: Some(e.to_string()), - orion_log_file: None, - }; - (StatusCode::INTERNAL_SERVER_ERROR, Json(response)).into_response() + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(WebhookResponse { + status: "error".to_string(), + vm_id: None, + domain: Some(domain), + phase: None, + error: Some(e.to_string()), + orion_log_file: None, + }), + ) + .into_response() } }; } - // Async path: return 202 immediately, provision in background. let state_clone = state.clone(); let vm_id_for_task = vm_id.clone(); + // Async path: return 202 immediately, provision in background. tokio::spawn(async move { let result = tokio::task::spawn_blocking(move || { let rt = tokio::runtime::Handle::current(); rt.block_on(orion_deployer::handle_update( &state_clone, + &domain_for_task, &label, &vm_id_for_task, target_config, @@ -288,13 +415,18 @@ pub async fn webhook_post_handler( } }); - let response = WebhookResponse { - status: "provisioning".to_string(), - vm_id: Some(vm_id), - error: None, - orion_log_file: None, - }; - (StatusCode::ACCEPTED, Json(response)).into_response() + ( + StatusCode::ACCEPTED, + Json(WebhookResponse { + status: "provisioning".to_string(), + vm_id: Some(vm_id), + domain: Some(domain), + phase: Some("provisioning".to_string()), + error: None, + orion_log_file: None, + }), + ) + .into_response() } /// GET /health @@ -305,31 +437,65 @@ pub async fn health_handler() -> Json { })) } -/// GET /status -pub async fn status_handler(State(state): State>) -> Json { - match orion_deployer::get_status(&state).await { - Some(vm) => { - let phase = vm.phase.as_str(); - let uptime_secs = if vm.phase == crate::state::VmPhase::Running { - Some(vm.created_at.elapsed().as_secs()) - } else { - None - }; +/// GET /status — list all VMs (optional ?domain= filter). +#[derive(Debug, Deserialize, Default)] +pub struct StatusQuery { + pub domain: Option, + pub vm_id: Option, +} + +pub async fn status_handler( + State(state): State>, + Query(q): Query, +) -> Json { + if let Some(id) = q.vm_id.as_deref() { + return match orion_deployer::get_status_by_id(&state, id).await { + Some(vm) => Json(vm_json(&vm)), + None => Json(serde_json::json!({ + "status": "no_vm", + "phase": "no_vm", + "vm_id": id + })), + }; + } + if let Some(domain) = q.domain.as_deref() { + return match orion_deployer::get_status_by_domain(&state, domain).await { + Some(vm) => Json(vm_json(&vm)), + None => Json(serde_json::json!({ + "status": "no_vm", + "phase": "no_vm", + "domain": domain, + "vm_id": null + })), + }; + } + + let list = orion_deployer::get_status(&state).await; + let vms: Vec<_> = list.iter().map(vm_json).collect(); + Json(serde_json::json!({ + "status": "ok", + "count": vms.len(), + "vms": vms + })) +} + +/// GET /vms/{id} +pub async fn vm_by_id_handler( + State(state): State>, + Path(id): Path, +) -> impl IntoResponse { + match orion_deployer::get_status_by_id(&state, &id).await { + Some(vm) => (StatusCode::OK, Json(vm_json(&vm))).into_response(), + None => ( + StatusCode::NOT_FOUND, Json(serde_json::json!({ - "status": phase, - "phase": phase, - "vm_id": vm.id, - "vm_ip": vm.ip, - "uptime_secs": uptime_secs, - "log_file": vm.log_file, - "error": vm.error - })) - } - None => Json(serde_json::json!({ - "status": "no_vm", - "phase": "no_vm", - "vm_id": null - })), + "status": "no_vm", + "phase": "no_vm", + "vm_id": id, + "error": "VM not found" + })), + ) + .into_response(), } } @@ -409,8 +575,18 @@ fn strip_ansi(text: &str) -> String { } /// GET /scorpio/status - Check Scorpio mount status and directories -pub async fn scorpio_status_handler(State(state): State>) -> impl IntoResponse { - match orion_deployer::get_scorpio_status(&state).await { +#[derive(Debug, Deserialize, Default)] +pub struct VmSelectQuery { + pub domain: Option, + pub vm_id: Option, +} + +pub async fn scorpio_status_handler( + State(state): State>, + Query(q): Query, +) -> impl IntoResponse { + let key = q.domain.or(q.vm_id); + match orion_deployer::get_scorpio_status(&state, key.as_deref()).await { Ok(status) => (StatusCode::OK, Json(status)).into_response(), Err(e) => { let response = serde_json::json!({ @@ -423,15 +599,19 @@ pub async fn scorpio_status_handler(State(state): State>) -> impl } /// GET /scorpio/config - Read scorpio.toml content from VM -pub async fn scorpio_config_handler(State(state): State>) -> impl IntoResponse { - let machine = match state.get_machine().await { - Some(m) => m, - None => { +pub async fn scorpio_config_handler( + State(state): State>, + Query(q): Query, +) -> impl IntoResponse { + let key = q.domain.or(q.vm_id); + let machine = match orion_deployer::resolve_machine_for_handlers(&state, key.as_deref()).await { + Ok(m) => m, + Err(e) => { return ( StatusCode::INTERNAL_SERVER_ERROR, Json(serde_json::json!({ "status": "error", - "error": "No VM is currently running" + "error": e.to_string() })), ) .into_response(); @@ -465,46 +645,104 @@ pub async fn scorpio_config_handler(State(state): State>) -> impl } } -/// POST /shutdown - Shutdown VM only, server keeps running -pub async fn shutdown_handler(State(state): State>) -> impl IntoResponse { - tracing::info!("[http-shutdown] Received shutdown request via HTTP"); +/// POST /shutdown — stop one VM; require `?domain=` or `?vm_id=` +/// (use `POST /shutdown/all` to stop every tracked VM). Server keeps running. +#[derive(Debug, Deserialize, Default)] +pub struct ShutdownQuery { + pub domain: Option, + pub vm_id: Option, +} + +pub async fn shutdown_handler( + State(state): State>, + Query(q): Query, +) -> impl IntoResponse { + tracing::info!( + "[http-shutdown] Received shutdown request domain={:?} vm_id={:?}", + q.domain, + q.vm_id + ); // Serialize with `handle_update`. Without this guard, /shutdown can // run between an in-flight create's `KeepAliveMachine::new` and its - // `state.set_vm`, see an empty state, return success, and leave the - // freshly-spawned qemu untracked once /webhook publishes it. + // `state.set_vm`, see an empty domain slot, return success, and leave + // the freshly-spawned qemu untracked once /webhook publishes it. let _update_guard = state.lock_update().await; - if let Some(machine) = state.get_machine().await { - tracing::info!("[http-shutdown] VM found, calling shutdown..."); - match machine.shutdown().await { - Ok(_) => tracing::info!("[http-shutdown] VM shutdown completed successfully"), - Err(e) => tracing::error!("[http-shutdown] VM shutdown failed: {}", e), + let domain = if let Some(d) = q.domain { + d + } else if let Some(id) = q.vm_id { + match state.domain_for_vm_id(&id).await { + Some(d) => d, + None => { + return ( + StatusCode::NOT_FOUND, + Json(serde_json::json!({ + "status": "error", + "error": format!("VM '{}' not found", id) + })), + ) + .into_response(); + } } } else { - tracing::info!("[http-shutdown] No VM running"); + return ( + StatusCode::BAD_REQUEST, + Json(serde_json::json!({ + "status": "error", + "error": "Specify ?domain= or ?vm_id= (or POST /shutdown/all)" + })), + ) + .into_response(); + }; + + if let Err(e) = orion_deployer::shutdown_domain(&state, &domain).await { + tracing::error!("[http-shutdown] failed: {e}"); } - state.clear_vm().await; - - // Belt-and-suspenders: reap any orphan qemu processes that may have - // escaped tracking (e.g. spawned by a previous crashed run, or - // mid-init when a prior shutdown raced). Cheap and idempotent. - let _ = tokio::process::Command::new("pkill") - .args(["-9", "-f", "qemu-system-x86"]) - .output() - .await; // Disk-side cleanup: qlean only removes the run dir from `Machine::drop`, // which doesn't run on SIGKILL/abort. Sweep any orphaned overlay/seed // files so /shutdown actually frees the VM's disk footprint, not just - // its processes. + // its processes. (No host-wide pkill — other domains must stay up.) vm_cleanup::sweep_stale_runs().await; - let response = serde_json::json!({ - "status": "ok", - "message": "VM stopped, server is still running" - }); - (StatusCode::OK, Json(response)).into_response() + ( + StatusCode::OK, + Json(serde_json::json!({ + "status": "ok", + "message": format!("VM for domain '{domain}' stopped"), + "domain": domain + })), + ) + .into_response() +} + +/// POST /shutdown/all — stop every tracked VM (ops only). Server keeps running. +pub async fn shutdown_all_handler(State(state): State>) -> impl IntoResponse { + tracing::info!("[http-shutdown] Received shutdown/all request"); + let _update_guard = state.lock_update().await; + let machines = state.take_all_machines().await; + for (info, machine) in machines { + tracing::info!( + "[http-shutdown] Shutting down {} ({})", + info.id, + info.domain + ); + machine.shutdown().await.ok(); + } + // Belt-and-suspenders: reap any orphan qemu listed under runs/ that may + // have escaped tracking (racing create, prior crash). Scoped to this + // XDG data tree — not a host-wide pkill. + vm_cleanup::reap_qemu_from_runs().await; + vm_cleanup::sweep_stale_runs().await; + ( + StatusCode::OK, + Json(serde_json::json!({ + "status": "ok", + "message": "All VMs stopped" + })), + ) + .into_response() } /// Number of trailing lines to send to the client on the first SSE tick. @@ -587,11 +825,15 @@ fn hash_line(line: &str) -> u64 { } /// GET /logs/orion/stream - SSE stream for real-time log viewing. +/// /// First tick sends the last `INITIAL_TAIL_LINES` lines, then only newly /// appended lines on each subsequent tick. +/// Multi-VM: pass `?domain=` or `?vm_id=` to select which runner's logs to stream. pub async fn logs_stream_handler( State(state): State>, + Query(q): Query, ) -> Sse>> { + let key = q.domain.or(q.vm_id); let stream = async_stream::stream! { let mut ticker = interval(std::time::Duration::from_secs(1)); let mut journal_cursor = LogCursor::default(); @@ -600,7 +842,13 @@ pub async fn logs_stream_handler( loop { ticker.tick().await; - let snapshot = match orion_deployer::get_live_logs_since(&state, orion_log_offset).await { + let snapshot = match orion_deployer::get_live_logs_since( + &state, + key.as_deref(), + orion_log_offset, + ) + .await + { Ok(snapshot) => snapshot, Err(e) => { yield Ok(Event::default().data(format!("Error: {}", e))); diff --git a/orion-scheduler/src/main.rs b/orion-scheduler/src/main.rs index 002417eea..6655f7d7c 100644 --- a/orion-scheduler/src/main.rs +++ b/orion-scheduler/src/main.rs @@ -14,15 +14,16 @@ use tokio::signal::{ctrl_c, unix::SignalKind}; use tower_http::cors::{Any, CorsLayer}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; -/// Gracefully shutdown VM and clear state on service termination signals. +/// Gracefully shutdown all tracked VMs on service termination signals. /// /// Acquires the update lock with a short timeout so a slow in-flight /// `/webhook` can't keep us racing forever (systemd would eventually -/// SIGKILL us and leave an orphan qemu). When the lock can't be taken -/// in time, we still proceed and rely on the pkill safety net below -/// to reap any qemu processes the racing create may have spawned. -async fn shutdown_vm(state: &AppState) { - tracing::info!("[shutdown] Initiating VM shutdown"); +/// SIGKILL us and leave orphan qemu). When the lock can't be taken +/// in time, we still proceed and rely on the run-dir qemu reap below +/// for processes that escaped tracking (no host-wide `pkill`, so other +/// domains / other schedulers on the same host are not touched). +async fn shutdown_all_vms(state: &AppState) { + tracing::info!("[shutdown] Initiating shutdown of all VMs"); let _guard = state .try_lock_update(std::time::Duration::from_secs(10)) @@ -30,36 +31,32 @@ async fn shutdown_vm(state: &AppState) { if _guard.is_none() { tracing::warn!( "[shutdown] timed out waiting for update lock; \ - proceeding and relying on pkill safety net" + proceeding with tracked machines and run-dir reap" ); } - if let Some(machine) = state.get_machine().await { - tracing::info!("[shutdown] VM found, calling shutdown..."); - match machine.shutdown().await { - Ok(_) => tracing::info!("[shutdown] VM shutdown completed successfully"), - Err(e) => tracing::error!("[shutdown] VM shutdown failed: {}", e), - } + let machines = state.take_all_machines().await; + if machines.is_empty() { + tracing::info!("[shutdown] No tracked VMs"); } else { - tracing::info!("[shutdown] No VM running"); + for (info, machine) in machines { + tracing::info!( + "[shutdown] Shutting down {} (domain={})", + info.id, + info.domain + ); + match machine.shutdown().await { + Ok(_) => tracing::info!("[shutdown] {} shut down OK", info.id), + Err(e) => tracing::error!("[shutdown] {} failed: {}", info.id, e), + } + } } - state.clear_vm().await; - // Reap any qemu process that escaped tracking — racing creates whose + // Reap any qemu that escaped tracking — racing creates whose // KeepAliveMachine never made it into `state`, or processes left over - // from a previous crashed run. Matches the same pattern we run at - // startup so the next run begins clean even if SIGKILL hits us next. - let pkill = tokio::process::Command::new("pkill") - .args(["-9", "-f", "qemu-system-x86"]) - .output() - .await; - match pkill { - Ok(out) if out.status.success() => { - tracing::warn!("[shutdown] pkill reaped leftover qemu process(es)"); - } - Ok(_) => tracing::info!("[shutdown] pkill found no leftover qemu"), - Err(e) => tracing::error!("[shutdown] pkill failed: {e}"), - } + // from a previous crashed run. Scoped to this process's qlean runs/ + // (via qemu.pid), not a host-wide pkill. + vm_cleanup::reap_qemu_from_runs().await; // Disk-side cleanup: even if Machine::drop ran, racing/aborted creates // can have left ~0.5–3 GB of overlay/seed on disk. Sweep here so we @@ -81,16 +78,14 @@ async fn main() -> anyhow::Result<()> { tracing::info!("Starting orion-scheduler service"); - // Cleanup any residual processes from previous runs, then sweep their - // on-disk run directories. The pkill alone can leave 0.5–3 GB per VM - // behind under ~/.local/share/qlean/runs/ because qlean only deletes - // those dirs from `Machine::drop`, which never runs on SIGKILL/abort. - tracing::info!("[startup] Checking for residual QEMU processes"); - tokio::process::Command::new("pkill") - .args(["-9", "-f", "qemu-system-x86"]) - .output() - .await - .ok(); + // Cleanup residual qemu from previous runs (scoped to this XDG data tree), + // then sweep on-disk run dirs. Unlike the old host-wide `pkill -f qemu`, + // this only touches pids recorded under ~/.local/share/qlean/runs/ so + // concurrent VMs owned by other domains/processes are not killed. qlean + // only deletes those dirs from `Machine::drop`, which never runs on + // SIGKILL/abort — leftover overlays are ~0.5–3 GB each. + tracing::info!("[startup] Reaping stale qemu from qlean runs/"); + vm_cleanup::reap_qemu_from_runs().await; vm_cleanup::sweep_stale_runs().await; // Load target configuration. @@ -118,8 +113,9 @@ async fn main() -> anyhow::Result<()> { let config = config::Config::load(&config_path).await?; let config = Arc::new(tokio::sync::RwLock::new(config)); tracing::info!( - "[startup] Config loaded, default_image path: {}", - config.read().await.default_image().image_path + "[startup] Config loaded, default_image path: {}, max_vms: {:?}", + config.read().await.default_image().image_path, + config.read().await.max_vms() ); // Create shared state @@ -137,6 +133,7 @@ async fn main() -> anyhow::Result<()> { ) .route("/health", axum::routing::get(handlers::health_handler)) .route("/status", axum::routing::get(handlers::status_handler)) + .route("/vms/{id}", axum::routing::get(handlers::vm_by_id_handler)) .route( "/logs/orion/stream", axum::routing::get(handlers::logs_stream_handler), @@ -150,6 +147,10 @@ async fn main() -> anyhow::Result<()> { axum::routing::get(handlers::scorpio_config_handler), ) .route("/shutdown", axum::routing::post(handlers::shutdown_handler)) + .route( + "/shutdown/all", + axum::routing::post(handlers::shutdown_all_handler), + ) .layer( CorsLayer::new() .allow_origin(Any) @@ -158,13 +159,13 @@ async fn main() -> anyhow::Result<()> { ) .with_state(state.clone()); - // Start server - let addr = "0.0.0.0:8080"; + // Start server (`LISTEN_ADDR` overrides the default for multi-process setups) + let addr = std::env::var("LISTEN_ADDR").unwrap_or_else(|_| "0.0.0.0:8080".to_string()); tracing::info!("[startup] Listening on {}", addr); - let listener = tokio::net::TcpListener::bind(addr).await?; + let listener = tokio::net::TcpListener::bind(&addr).await?; - // Handle termination signals: stop VM and server + // Handle termination signals: stop all tracked VMs and the server let term_shutdown_state = state.clone(); let term_shutdown_signal = async move { if let Some(()) = tokio::signal::unix::signal(SignalKind::terminate()) @@ -173,7 +174,7 @@ async fn main() -> anyhow::Result<()> { .await { tracing::info!("[shutdown] Received SIGTERM"); - shutdown_vm(&term_shutdown_state).await; + shutdown_all_vms(&term_shutdown_state).await; } }; @@ -185,23 +186,23 @@ async fn main() -> anyhow::Result<()> { .await { tracing::info!("[shutdown] Received SIGQUIT"); - shutdown_vm(&quit_shutdown_state).await; + shutdown_all_vms(&quit_shutdown_state).await; } }; - // Handle Ctrl+C: stop VM and server + // Handle Ctrl+C: stop all tracked VMs and the server let ctrl_c_shutdown_state = state.clone(); let ctrl_c_signal = async move { match ctrl_c().await { Ok(()) => { tracing::info!("[shutdown] Received SIGINT (Ctrl+C)"); - shutdown_vm(&ctrl_c_shutdown_state).await; + shutdown_all_vms(&ctrl_c_shutdown_state).await; } Err(e) => tracing::error!("[shutdown] Ctrl+C handler error: {}", e), } }; - tracing::info!("[startup] Server running. Use /shutdown to stop VM only"); + tracing::info!("[startup] Server running. Use /shutdown?domain=… to stop one VM"); axum::serve(listener, app) .with_graceful_shutdown(async { tokio::select! { diff --git a/orion-scheduler/src/orion_deployer.rs b/orion-scheduler/src/orion_deployer.rs index 8d9034b47..4d947f462 100644 --- a/orion-scheduler/src/orion_deployer.rs +++ b/orion-scheduler/src/orion_deployer.rs @@ -32,32 +32,67 @@ pub fn validate_runner_env( Ok(()) } +/// Extract uniqueness domain (host) from a `server_ws` URL. +pub fn domain_from_server_ws(server_ws: &str) -> Result { + let url = url::Url::parse(server_ws.trim()) + .map_err(|e| anyhow::anyhow!("invalid server_ws URL: {e}"))?; + let host = url + .host_str() + .ok_or_else(|| anyhow::anyhow!("server_ws has no host"))? + .to_string(); + if host.is_empty() { + return Err(anyhow::anyhow!("server_ws host is empty")); + } + Ok(host) +} + +/// Shut down and clear only the VM for `domain`, if any. +pub async fn shutdown_domain(state: &AppState, domain: &str) -> Result<()> { + if let Some((info, machine)) = state.take_machine_by_domain(domain).await { + info!( + "[orion-deploy] Shutting down VM {} for domain {}", + info.id, domain + ); + machine.shutdown().await.ok(); + } else { + state.clear_domain(domain).await; + } + Ok(()) +} + /// Handle the update request (keep_alive mode). VM stays running after this call. /// +/// Only replaces/manages the slot for `domain`. Does not touch other domains. /// `vm_name` must be pre-generated by the caller so the webhook can return it /// immediately before this function completes. pub async fn handle_update( state: &AppState, + domain: &str, label: &str, vm_name: &str, target_config: TargetConfig, image_params: ImageParams, ) -> Result { - info!("Handling update request, label: {}, vm: {}", label, vm_name); + info!( + "Handling update request, domain: {}, label: {}, vm: {}", + domain, label, vm_name + ); let _update_guard = state.lock_update().await; - info!("[orion-deploy] Acquired update lock for label: {}", label); + info!("[orion-deploy] Acquired update lock for domain: {}", domain); - let result = handle_update_inner(state, label, vm_name, target_config, image_params).await; + let result = + handle_update_inner(state, domain, label, vm_name, target_config, image_params).await; if let Err(ref e) = result { tracing::error!("[orion-deploy] Update failed for {}: {:?}", vm_name, e); - state.set_vm_failed(vm_name, e.to_string()).await; + state.set_vm_failed(domain, vm_name, e.to_string()).await; } result } async fn handle_update_inner( state: &AppState, + domain: &str, label: &str, vm_name: &str, target_config: TargetConfig, @@ -70,16 +105,19 @@ async fn handle_update_inner( let ssh_public_key_path = config.ssh_public_key_path().to_string(); drop(config); - if let Some(existing_vm) = state.get_vm().await { - info!("Found existing VM {}, shutting down", existing_vm.id); - if let Some(machine) = state.get_machine().await { - machine.shutdown().await.ok(); - } - state.clear_vm().await; + // Only shut down the same domain (Failed/replace path). + if let Some(existing) = state.get_vm_by_domain(domain).await { + info!( + "Found existing VM {} for domain {}, shutting down", + existing.id, domain + ); + shutdown_domain(state, domain).await?; } let provisioning_info = VmInfo { id: vm_name.to_string(), + domain: domain.to_string(), + target: label.to_string(), phase: VmPhase::Provisioning, ip: None, created_at: std::time::Instant::now(), @@ -110,6 +148,8 @@ async fn handle_update_inner( let vm_info = VmInfo { id: vm_name.to_string(), + domain: domain.to_string(), + target: label.to_string(), phase: VmPhase::Running, ip: vm_ip, created_at: std::time::Instant::now(), @@ -118,7 +158,10 @@ async fn handle_update_inner( }; state.set_vm(vm_info, machine).await; - info!("Update completed successfully for label: {}", label); + info!( + "Update completed successfully for domain: {}, label: {}", + domain, label + ); Ok(vm_name.to_string()) } @@ -191,12 +234,10 @@ pub struct LiveLogSnapshot { pub async fn get_live_logs_since( state: &AppState, + domain_or_vm_id: Option<&str>, orion_log_offset: u64, ) -> Result { - let machine = state - .get_machine() - .await - .ok_or_else(|| anyhow::anyhow!("No VM is currently running"))?; + let machine = resolve_machine(state, domain_or_vm_id).await?; let output = machine .exec("journalctl -u orion-runner --no-pager -n 200 2>&1") @@ -239,15 +280,67 @@ pub async fn get_live_logs_since( }) } -pub async fn get_status(state: &AppState) -> Option { - state.get_vm().await +async fn resolve_machine( + state: &AppState, + domain_or_vm_id: Option<&str>, +) -> Result { + match domain_or_vm_id { + Some(key) => { + if let Some(m) = state.get_machine_by_domain(key).await { + return Ok(m); + } + if let Some(m) = state.get_machine_by_id(key).await { + return Ok(m); + } + Err(anyhow::anyhow!("No running VM for key '{}'", key)) + } + None => { + let list = state.list_vms().await; + let running: Vec<_> = list + .into_iter() + .filter(|v| v.phase == VmPhase::Running) + .collect(); + if running.len() == 1 { + state + .get_machine_by_domain(&running[0].domain) + .await + .ok_or_else(|| anyhow::anyhow!("No VM machine handle available")) + } else if running.is_empty() { + Err(anyhow::anyhow!("No VM is currently running")) + } else { + Err(anyhow::anyhow!( + "Multiple VMs running; specify ?domain= or ?vm_id=" + )) + } + } + } } -pub async fn get_scorpio_status(state: &AppState) -> Result { - let machine = state - .get_machine() - .await - .ok_or_else(|| anyhow::anyhow!("No VM is currently running"))?; +/// Public wrapper for HTTP handlers that need a machine by domain/vm_id. +pub async fn resolve_machine_for_handlers( + state: &AppState, + domain_or_vm_id: Option<&str>, +) -> Result { + resolve_machine(state, domain_or_vm_id).await +} + +pub async fn get_status(state: &AppState) -> Vec { + state.list_vms().await +} + +pub async fn get_status_by_id(state: &AppState, id: &str) -> Option { + state.get_vm_by_id(id).await +} + +pub async fn get_status_by_domain(state: &AppState, domain: &str) -> Option { + state.get_vm_by_domain(domain).await +} + +pub async fn get_scorpio_status( + state: &AppState, + domain_or_vm_id: Option<&str>, +) -> Result { + let machine = resolve_machine(state, domain_or_vm_id).await?; info!("[scorpio] Checking mount status and directories"); @@ -381,6 +474,18 @@ mod tests { use super::*; use crate::handlers::ImageParams; + #[test] + fn domain_from_server_ws_extracts_host() { + assert_eq!( + domain_from_server_ws("wss://orion.xuanwu.openatom.cn/ws").unwrap(), + "orion.xuanwu.openatom.cn" + ); + assert_eq!( + domain_from_server_ws("ws://orion.gitmega.com:8080/ws").unwrap(), + "orion.gitmega.com" + ); + } + #[test] fn build_image_spec_from_url_params() { let params = ImageParams { diff --git a/orion-scheduler/src/state.rs b/orion-scheduler/src/state.rs index cd2aaeaa1..ec33741d8 100644 --- a/orion-scheduler/src/state.rs +++ b/orion-scheduler/src/state.rs @@ -1,4 +1,4 @@ -use std::{sync::Arc, time::Duration}; +use std::{collections::HashMap, sync::Arc, time::Duration}; use tokio::sync::{Mutex, MutexGuard, RwLock}; @@ -22,10 +22,14 @@ impl VmPhase { } } -/// Represents the current state of the VM +/// Represents the current state of one VM (keyed by domain in AppState). #[derive(Debug, Clone)] pub struct VmInfo { pub id: String, + /// Host parsed from `server_ws` (uniqueness key). + pub domain: String, + /// Optional label from webhook `target` field. + pub target: String, pub phase: VmPhase, pub ip: Option, pub created_at: std::time::Instant, @@ -35,25 +39,30 @@ pub struct VmInfo { pub error: Option, } -/// Global state for tracking VM lifecycle +pub struct VmEntry { + pub info: VmInfo, + pub machine: Option, +} + +/// Global state for tracking multiple VMs (one per domain). pub struct AppState { - pub vm: Arc>>, - pub machine: Arc>>, + /// key = domain (`server_ws` host) + pub vms: Arc>>, pub config: SharedConfig, /// Single-flight mutex guarding the full VM update sequence - /// (shutdown existing VM → create new VM → publish to state). - /// Without this, two concurrent /webhook calls can both pass the - /// existing-VM check before either stores its new machine, leaking + /// (shutdown existing slot for a domain → create new VM → publish to state). + /// Without this, two concurrent /webhook calls for the same domain can both + /// pass the conflict check before either stores its new machine, leaking /// the earlier qemu process out of `state` and out of `/shutdown`'s reach. + /// Coarse (global) for MVP; can later become a per-domain Mutex. update_lock: Arc>, } impl AppState { - /// Create a new AppState with empty VM and machine slots + /// Create a new AppState with an empty VM map. pub fn new(config: SharedConfig) -> Self { Self { - vm: Arc::new(RwLock::new(None)), - machine: Arc::new(RwLock::new(None)), + vms: Arc::new(RwLock::new(HashMap::new())), config, update_lock: Arc::new(Mutex::new(())), } @@ -65,8 +74,8 @@ impl AppState { /// /// `/shutdown` and signal-triggered teardown must also hold this guard /// to avoid running between an in-flight create's - /// `KeepAliveMachine::new` and `set_vm`, which would otherwise see an - /// empty state and leave the freshly-spawned qemu untracked. + /// `KeepAliveMachine::new` and `set_vm`, which would otherwise miss the + /// freshly-spawned qemu and leave it untracked. pub async fn lock_update(&self) -> MutexGuard<'_, ()> { self.update_lock.lock().await } @@ -74,64 +83,125 @@ impl AppState { /// Like `lock_update`, but bounded so signal handlers don't hang the /// process behind a multi-minute create. Returns `None` if the lock /// could not be acquired within `timeout`; callers must then fall back - /// to a force-kill safety net. + /// to the run-dir qemu reap safety net. pub async fn try_lock_update(&self, timeout: Duration) -> Option> { tokio::time::timeout(timeout, self.update_lock.lock()) .await .ok() } + pub async fn list_vms(&self) -> Vec { + let vms = self.vms.read().await; + vms.values().map(|e| e.info.clone()).collect() + } + + pub async fn vm_count(&self) -> usize { + self.vms.read().await.len() + } + + pub async fn get_vm_by_domain(&self, domain: &str) -> Option { + let vms = self.vms.read().await; + vms.get(domain).map(|e| e.info.clone()) + } + + pub async fn get_vm_by_id(&self, id: &str) -> Option { + let vms = self.vms.read().await; + vms.values() + .find(|e| e.info.id == id) + .map(|e| e.info.clone()) + } + + pub async fn get_machine_by_domain(&self, domain: &str) -> Option { + let vms = self.vms.read().await; + vms.get(domain).and_then(|e| e.machine.clone()) + } + + pub async fn get_machine_by_id(&self, id: &str) -> Option { + let vms = self.vms.read().await; + vms.values() + .find(|e| e.info.id == id) + .and_then(|e| e.machine.clone()) + } + + /// Domain for a vm_id, if tracked. + pub async fn domain_for_vm_id(&self, id: &str) -> Option { + let vms = self.vms.read().await; + vms.values() + .find(|e| e.info.id == id) + .map(|e| e.info.domain.clone()) + } + /// Register a VM in provisioning state (no machine handle yet). pub async fn set_vm_provisioning(&self, info: VmInfo) { - let mut vm = self.vm.write().await; - let mut m = self.machine.write().await; - *vm = Some(info); - *m = None; + let domain = info.domain.clone(); + let mut vms = self.vms.write().await; + vms.insert( + domain, + VmEntry { + info, + machine: None, + }, + ); } - /// Set VM info and machine reference together atomically. - /// Both write locks are held simultaneously so concurrent readers - /// never observe a half-published state (e.g. `vm = Some` with - /// `machine = None`), which previously allowed a shutdown racing - /// `set_vm` to clear the entry while the qemu kept running. + /// Set VM info and machine reference together for one domain. + /// A single map write makes readers never observe a half-published entry + /// (e.g. Running with `machine = None`), which previously allowed a + /// shutdown racing `set_vm` to clear the slot while qemu kept running. pub async fn set_vm(&self, info: VmInfo, machine: KeepAliveMachine) { - let mut vm = self.vm.write().await; - let mut m = self.machine.write().await; - *vm = Some(info); - *m = Some(machine); - } - - /// Mark the current VM as failed, clearing any machine handle. - pub async fn set_vm_failed(&self, id: &str, error: String) { - let mut vm = self.vm.write().await; - let mut m = self.machine.write().await; - if let Some(info) = vm.as_mut() - && info.id == id + let domain = info.domain.clone(); + let mut vms = self.vms.write().await; + vms.insert( + domain, + VmEntry { + info, + machine: Some(machine), + }, + ); + } + + /// Mark the VM for `domain` as failed (if `id` still matches), clearing + /// any machine handle so later cleanup does not double-shutdown. + pub async fn set_vm_failed(&self, domain: &str, id: &str, error: String) { + let mut vms = self.vms.write().await; + if let Some(entry) = vms.get_mut(domain) + && entry.info.id == id { - info.phase = VmPhase::Failed; - info.error = Some(error); + entry.info.phase = VmPhase::Failed; + entry.info.error = Some(error); + entry.machine = None; } - *m = None; } - /// Clear both VM info and machine reference atomically. - pub async fn clear_vm(&self) { - let mut vm = self.vm.write().await; - let mut m = self.machine.write().await; - *vm = None; - *m = None; + /// Remove a domain slot and return its machine for shutdown. + /// If the slot had no machine (Provisioning/Failed tombstone), the map + /// entry is still removed and `None` is returned. + pub async fn take_machine_by_domain(&self, domain: &str) -> Option<(VmInfo, KeepAliveMachine)> { + let mut vms = self.vms.write().await; + let entry = vms.remove(domain)?; + match entry.machine { + Some(m) => Some((entry.info, m)), + None => None, + } } - /// Get a clone of the current VM info if any - pub async fn get_vm(&self) -> Option { - let vm = self.vm.read().await; - vm.clone() + /// Clear a domain slot without shutting down a machine (caller already did, + /// or there was only a tombstone). + pub async fn clear_domain(&self, domain: &str) { + let mut vms = self.vms.write().await; + vms.remove(domain); } - /// Get a clone of the current machine reference if any - pub async fn get_machine(&self) -> Option { - let m = self.machine.read().await; - m.clone() + /// Drain every tracked machine for process-exit / `/shutdown/all` teardown. + pub async fn take_all_machines(&self) -> Vec<(VmInfo, KeepAliveMachine)> { + let mut vms = self.vms.write().await; + let mut out = Vec::new(); + for (_, entry) in vms.drain() { + if let Some(m) = entry.machine { + out.push((entry.info, m)); + } + } + out } } @@ -139,8 +209,7 @@ impl AppState { mod tests { use super::*; - #[tokio::test] - async fn test_state_set_clear() { + fn test_state() -> AppState { let config = Arc::new(tokio::sync::RwLock::new(crate::config::Config::new( "/tmp".to_string(), "/tmp/orion".to_string(), @@ -148,38 +217,56 @@ mod tests { "/tmp/ssh_key.pub".to_string(), Default::default(), ))); - let state = AppState::new(config); - assert!(state.get_vm().await.is_none()); - assert!(state.get_machine().await.is_none()); + AppState::new(config) } - #[tokio::test] - async fn test_provisioning_to_failed() { - let config = Arc::new(tokio::sync::RwLock::new(crate::config::Config::new( - "/tmp".to_string(), - "/tmp/orion".to_string(), - "/tmp/orion".to_string(), - "/tmp/ssh_key.pub".to_string(), - Default::default(), - ))); - let state = AppState::new(config); - let info = VmInfo { - id: "orion-vm-1".to_string(), - phase: VmPhase::Provisioning, + fn sample_info(domain: &str, id: &str, phase: VmPhase) -> VmInfo { + VmInfo { + id: id.to_string(), + domain: domain.to_string(), + target: "t".to_string(), + phase, ip: None, created_at: std::time::Instant::now(), log_file: None, error: None, - }; - state.set_vm_provisioning(info).await; - let vm = state.get_vm().await.unwrap(); - assert_eq!(vm.phase, VmPhase::Provisioning); - assert!(state.get_machine().await.is_none()); + } + } + #[tokio::test] + async fn two_domains_coexist() { + let state = test_state(); + state + .set_vm_provisioning(sample_info("orion.a.com", "vm-a", VmPhase::Provisioning)) + .await; + state + .set_vm_provisioning(sample_info("orion.b.com", "vm-b", VmPhase::Provisioning)) + .await; + assert_eq!(state.vm_count().await, 2); + assert_eq!( + state.get_vm_by_domain("orion.a.com").await.unwrap().id, + "vm-a" + ); + assert_eq!( + state.get_vm_by_id("vm-b").await.unwrap().domain, + "orion.b.com" + ); + } + + #[tokio::test] + async fn provisioning_to_failed() { + let state = test_state(); + state + .set_vm_provisioning(sample_info( + "orion.a.com", + "orion-vm-1", + VmPhase::Provisioning, + )) + .await; state - .set_vm_failed("orion-vm-1", "deploy failed".to_string()) + .set_vm_failed("orion.a.com", "orion-vm-1", "deploy failed".to_string()) .await; - let vm = state.get_vm().await.unwrap(); + let vm = state.get_vm_by_domain("orion.a.com").await.unwrap(); assert_eq!(vm.phase, VmPhase::Failed); assert_eq!(vm.error.as_deref(), Some("deploy failed")); } diff --git a/orion-scheduler/src/vm_cleanup.rs b/orion-scheduler/src/vm_cleanup.rs index a1a199932..6a0605456 100644 --- a/orion-scheduler/src/vm_cleanup.rs +++ b/orion-scheduler/src/vm_cleanup.rs @@ -9,8 +9,9 @@ //! gigabytes in practice. //! //! This module sweeps directories whose `qemu.pid` no longer maps to a live -//! process. It is safe to run at startup (after we pkill any stale qemu) and -//! at shutdown (after the same pkill), and it is idempotent. +//! process, and can kill still-live qemu listed in those pid files +//! (`reap_qemu_from_runs`) without a host-wide `pkill`. Safe at startup and +//! shutdown; idempotent. use std::path::{Path, PathBuf}; @@ -96,3 +97,65 @@ async fn is_run_alive(run_dir: &Path) -> bool { }; Path::new(&format!("/proc/{pid}")).exists() } + +/// Kill qemu processes listed in each run dir's `qemu.pid` (scoped to this +/// scheduler's XDG data tree). Prefer this over host-wide `pkill`. +pub async fn reap_qemu_from_runs() -> usize { + let Some(runs_dir) = qlean_runs_dir() else { + tracing::warn!("[reap] cannot resolve qlean runs dir; HOME unset"); + return 0; + }; + + let mut entries = match tokio::fs::read_dir(&runs_dir).await { + Ok(e) => e, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return 0, + Err(e) => { + tracing::warn!("[reap] read_dir {} failed: {e}", runs_dir.display()); + return 0; + } + }; + + let mut killed = 0usize; + loop { + let entry = match entries.next_entry().await { + Ok(Some(e)) => e, + Ok(None) => break, + Err(e) => { + tracing::warn!("[reap] read_dir iteration failed: {e}"); + break; + } + }; + let path = entry.path(); + let is_dir = entry.file_type().await.map(|t| t.is_dir()).unwrap_or(false); + if !is_dir { + continue; + } + let pid_file = path.join("qemu.pid"); + let Ok(contents) = tokio::fs::read_to_string(&pid_file).await else { + continue; + }; + let Ok(pid) = contents.trim().parse::() else { + continue; + }; + if !Path::new(&format!("/proc/{pid}")).exists() { + continue; + } + match tokio::process::Command::new("kill") + .args(["-9", &pid.to_string()]) + .output() + .await + { + Ok(out) if out.status.success() => { + tracing::warn!("[reap] killed qemu pid={pid} from {}", path.display()); + killed += 1; + } + Ok(_) => tracing::debug!("[reap] kill -9 {pid} returned non-zero"), + Err(e) => tracing::warn!("[reap] kill {pid} failed: {e}"), + } + } + + if killed > 0 { + tracing::warn!("[reap] killed {killed} qemu process(es) from runs/"); + } + killed +} diff --git a/orion-scheduler/target_config.json.template b/orion-scheduler/target_config.json.template index 2327e5ab9..ae1b306d0 100644 --- a/orion-scheduler/target_config.json.template +++ b/orion-scheduler/target_config.json.template @@ -3,6 +3,7 @@ "orion_source_dir": "/path/to/mega/orion", "orion_binary_path": "/path/to/mega/target/debug/orion", "ssh_public_key_path": "~/.ssh/orion_vm_access.pub", + "max_vms": 8, "default_image": { "image_path": "~/.local/share/qlean/images/debian-13-buck2/debian-13-buck2.qcow2", "image_digest": "sha256:753c28888c9d30fe4baef55c1d1dfa9a39431595eca940b7ad85d78d84f3d7a5",