From 07c9f2d652d1d77d3619d17429b6628078348701 Mon Sep 17 00:00:00 2001 From: loonghao Date: Thu, 28 May 2026 03:26:35 +0800 Subject: [PATCH] feat: package lightweight deployment profile --- Cargo.lock | 35 ++++ Cargo.toml | 2 +- README.md | 16 +- crates/farm-controller/Cargo.toml | 2 + crates/farm-controller/src/main.rs | 180 ++++++++++++++++- crates/farm-worker/Cargo.toml | 2 + crates/farm-worker/src/main.rs | 189 +++++++++++++++++- deploy/lightweight/controller.yaml | 7 + .../systemd/renderacre-controller.service | 18 ++ .../systemd/renderacre-worker.service | 18 ++ .../windows/install-controller-service.ps1 | 29 +++ .../windows/install-worker-service.ps1 | 29 +++ deploy/lightweight/worker.yaml | 9 + docs/architecture.md | 2 +- docs/deployment.md | 159 +++++++++++++++ 15 files changed, 675 insertions(+), 22 deletions(-) create mode 100644 deploy/lightweight/controller.yaml create mode 100644 deploy/lightweight/systemd/renderacre-controller.service create mode 100644 deploy/lightweight/systemd/renderacre-worker.service create mode 100644 deploy/lightweight/windows/install-controller-service.ps1 create mode 100644 deploy/lightweight/windows/install-worker-service.ps1 create mode 100644 deploy/lightweight/worker.yaml create mode 100644 docs/deployment.md diff --git a/Cargo.lock b/Cargo.lock index 918a37e..8649600 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -701,6 +701,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + [[package]] name = "httparse" version = "1.10.1" @@ -1099,6 +1105,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "mio" version = "1.2.0" @@ -1590,7 +1606,9 @@ dependencies = [ "axum", "clap", "farm-core", + "serde", "serde_json", + "serde_yaml", "tokio", "tower-http", "tracing", @@ -1621,7 +1639,9 @@ dependencies = [ "openjd-model", "openjd-sessions", "reqwest", + "serde", "serde_json", + "serde_yaml", "tempfile", "tokio", "tracing", @@ -2286,10 +2306,19 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "bitflags", "bytes", + "futures-core", "futures-util", "http", "http-body", + "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -2383,6 +2412,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" diff --git a/Cargo.toml b/Cargo.toml index 02f35fc..d22bb8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ serde_yaml = "0.9" tempfile = "3.23" thiserror = "2.0" tokio = { version = "1.48", features = ["macros", "net", "process", "rt-multi-thread", "signal", "time"] } -tower-http = { version = "0.6", features = ["cors", "trace"] } +tower-http = { version = "0.6", features = ["cors", "fs", "trace"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } uuid = { version = "1.18", features = ["serde", "v4"] } diff --git a/README.md b/README.md index 2775c12..6e0879e 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ cd dashboard npm run build ``` -Serve `dashboard/dist` through your internal web server or reverse proxy next to the controller API. +Serve `dashboard/dist` through the controller with `--dashboard-dir dashboard/dist`, or through your internal web server or reverse proxy next to the controller API. ## Command and DCC Examples @@ -339,7 +339,7 @@ artifact download, or worker log line is missing. Recommended first deployment shape: -- Run one controller per farm or queue: `renderacre-controller --bind 0.0.0.0:7878`. +- Run one controller per farm or queue: `renderacre-controller --config deploy/lightweight/controller.yaml`. - Run one worker process per render node: `renderacre-worker --controller http://controller-host:7878 --name --label app=blender`. - Put the controller behind a private network or authenticated reverse proxy. - Keep render executables and scripts on shared storage, then pass `PATH` parameters through OpenJD. @@ -347,14 +347,18 @@ Recommended first deployment shape: SQLite is the default durable profile for small deployments. Start the controller with `--storage sqlite --sqlite-path ` or set -`RFARM_STORAGE=sqlite` and `RFARM_SQLITE_PATH`. The scheduler API remains the -same for REST workers, dashboard reads, and Python submitters; future Postgres -or managed/cloud storage backends can replace the same storage boundary without -changing submitter contracts. +`RFARM_STORAGE=sqlite` and `RFARM_SQLITE_PATH`. Use `--dashboard-dir ` or +`RFARM_DASHBOARD_DIR` to serve a built dashboard from the controller in the +single-node profile. The scheduler API remains the same for REST workers, +dashboard reads, and Python submitters; future Postgres or managed/cloud storage +backends can replace the same storage boundary without changing submitter +contracts. Cloud-ready backend, artifact, worker identity, and scheduler extension contracts are documented in [docs/extension-contracts.md](docs/extension-contracts.md). +The one-command lightweight profile, service examples, and upgrade/backup +guidance are documented in [docs/deployment.md](docs/deployment.md). ## Release diff --git a/crates/farm-controller/Cargo.toml b/crates/farm-controller/Cargo.toml index bcfbade..dbaecdf 100644 --- a/crates/farm-controller/Cargo.toml +++ b/crates/farm-controller/Cargo.toml @@ -13,7 +13,9 @@ anyhow.workspace = true axum.workspace = true clap.workspace = true farm-core = { path = "../farm-core" } +serde.workspace = true serde_json.workspace = true +serde_yaml.workspace = true tokio.workspace = true tower-http.workspace = true tracing.workspace = true diff --git a/crates/farm-controller/src/main.rs b/crates/farm-controller/src/main.rs index bb81dee..aa9bafe 100644 --- a/crates/farm-controller/src/main.rs +++ b/crates/farm-controller/src/main.rs @@ -1,13 +1,17 @@ +#[cfg(test)] +use std::ffi::OsString; use std::net::SocketAddr; -use std::path::PathBuf; +use std::path::{Path as FsPath, PathBuf}; +use anyhow::Context; use axum::body::Body; use axum::extract::{Path, State}; use axum::http::{header, HeaderValue, StatusCode}; use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Json, Router}; -use clap::{Parser, ValueEnum}; +use clap::parser::ValueSource; +use clap::{ArgMatches, CommandFactory, FromArgMatches, Parser, ValueEnum}; use farm_core::{ AuditEvent, DashboardSnapshot, FarmError, FarmLogEntry, FarmMetrics, FarmStats, HealthComponent, HealthReport, HealthStatus, InMemoryScheduler, Job, JobId, JobPriorityUpdate, @@ -15,12 +19,16 @@ use farm_core::{ Task, TaskComplete, TaskId, TaskLease, TaskLeaseRenewal, TaskStarted, WorkerId, WorkerInfo, WorkerLogBatch, WorkerRegister, }; +use serde::Deserialize; use serde_json::json; use tower_http::cors::CorsLayer; +use tower_http::services::{ServeDir, ServeFile}; use tower_http::trace::TraceLayer; #[derive(Debug, Parser)] struct Args { + #[arg(long, env = "RFARM_CONFIG")] + config: Option, #[arg(long, env = "RFARM_BIND", default_value = "127.0.0.1:7878")] bind: SocketAddr, #[arg(long, env = "RFARM_LEASE_SECONDS", default_value_t = 120)] @@ -29,14 +37,94 @@ struct Args { storage: StorageBackend, #[arg(long, env = "RFARM_SQLITE_PATH", default_value = "renderacre.sqlite3")] sqlite_path: PathBuf, + #[arg(long, env = "RFARM_DASHBOARD_DIR")] + dashboard_dir: Option, } -#[derive(Debug, Clone, Copy, ValueEnum)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum, Deserialize)] +#[serde(rename_all = "snake_case")] enum StorageBackend { Memory, Sqlite, } +#[derive(Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct ControllerConfig { + bind: Option, + lease_seconds: Option, + storage: Option, + sqlite_path: Option, + dashboard_dir: Option, +} + +impl Args { + fn load() -> anyhow::Result { + Self::from_matches(Self::command().get_matches()) + } + + #[cfg(test)] + fn load_from(iter: I) -> anyhow::Result + where + I: IntoIterator, + T: Into + Clone, + { + let matches = Self::command().try_get_matches_from(iter)?; + Self::from_matches(matches) + } + + fn from_matches(matches: ArgMatches) -> anyhow::Result { + let mut args = Self::from_arg_matches(&matches)?; + let config = ControllerConfig::load(args.config.as_deref())?; + args.apply_config(config, &matches); + Ok(args) + } + + fn apply_config(&mut self, config: ControllerConfig, matches: &ArgMatches) { + if value_from_default(matches, "bind") { + if let Some(bind) = config.bind { + self.bind = bind; + } + } + if value_from_default(matches, "lease_seconds") { + if let Some(lease_seconds) = config.lease_seconds { + self.lease_seconds = lease_seconds; + } + } + if value_from_default(matches, "storage") { + if let Some(storage) = config.storage { + self.storage = storage; + } + } + if value_from_default(matches, "sqlite_path") { + if let Some(sqlite_path) = config.sqlite_path { + self.sqlite_path = sqlite_path; + } + } + if matches.value_source("dashboard_dir").is_none() { + if let Some(dashboard_dir) = config.dashboard_dir { + self.dashboard_dir = Some(dashboard_dir); + } + } + } +} + +impl ControllerConfig { + fn load(path: Option<&FsPath>) -> anyhow::Result { + let Some(path) = path else { + return Ok(Self::default()); + }; + let content = std::fs::read_to_string(path) + .with_context(|| format!("failed to read config file {}", path.display()))?; + serde_yaml::from_str(&content) + .with_context(|| format!("failed to parse config file {}", path.display())) + } +} + +fn value_from_default(matches: &ArgMatches, id: &str) -> bool { + matches.value_source(id) == Some(ValueSource::DefaultValue) +} + #[derive(Clone)] enum AppScheduler { Memory(InMemoryScheduler), @@ -295,7 +383,7 @@ async fn main() -> anyhow::Result<()> { ) .init(); - let args = Args::parse(); + let args = Args::load()?; let listener = tokio::net::TcpListener::bind(args.bind).await?; let config = SchedulerConfig { lease_ttl_seconds: args.lease_seconds, @@ -308,12 +396,15 @@ async fn main() -> anyhow::Result<()> { )?), }; tracing::info!("controller listening on http://{}", args.bind); - axum::serve(listener, app(scheduler)).await?; + if let Some(dashboard_dir) = args.dashboard_dir.as_ref() { + tracing::info!(path = %dashboard_dir.display(), "serving dashboard assets"); + } + axum::serve(listener, app(scheduler, args.dashboard_dir)).await?; Ok(()) } -fn app(scheduler: AppScheduler) -> Router { - Router::new() +fn app(scheduler: AppScheduler, dashboard_dir: Option) -> Router { + let router = Router::new() .route("/healthz", get(healthz)) .route("/readyz", get(healthz)) .route("/v1/health", get(healthz)) @@ -352,7 +443,14 @@ fn app(scheduler: AppScheduler) -> Router { ) .layer(CorsLayer::permissive()) .layer(TraceLayer::new_for_http()) - .with_state(scheduler) + .with_state(scheduler); + + if let Some(dashboard_dir) = dashboard_dir { + let index = dashboard_dir.join("index.html"); + router.fallback_service(ServeDir::new(dashboard_dir).fallback(ServeFile::new(index))) + } else { + router + } } async fn healthz(State(scheduler): State) -> Json { @@ -621,6 +719,65 @@ fn content_type_for(name: &str) -> &'static str { mod tests { use super::*; + #[test] + fn config_file_overrides_controller_defaults() { + let config_path = write_config( + "controller", + r#" +bind: 127.0.0.1:9001 +lease_seconds: 45 +storage: sqlite +sqlite_path: data/renderacre.sqlite3 +dashboard_dir: dashboard/dist +"#, + ); + + let args = Args::load_from([ + "renderacre-controller", + "--config", + config_path.to_str().unwrap(), + ]) + .expect("config should load"); + + assert_eq!(args.bind, "127.0.0.1:9001".parse::().unwrap()); + assert_eq!(args.lease_seconds, 45); + assert_eq!(args.storage, StorageBackend::Sqlite); + assert_eq!(args.sqlite_path, PathBuf::from("data/renderacre.sqlite3")); + assert_eq!(args.dashboard_dir, Some(PathBuf::from("dashboard/dist"))); + let _ = std::fs::remove_file(config_path); + } + + #[test] + fn command_line_values_override_controller_config() { + let config_path = write_config( + "controller-cli", + r#" +bind: 127.0.0.1:9001 +lease_seconds: 45 +storage: sqlite +sqlite_path: data/renderacre.sqlite3 +dashboard_dir: dashboard/dist +"#, + ); + + let args = Args::load_from([ + "renderacre-controller", + "--config", + config_path.to_str().unwrap(), + "--bind", + "127.0.0.1:9002", + "--storage", + "memory", + ]) + .expect("config should load"); + + assert_eq!(args.bind, "127.0.0.1:9002".parse::().unwrap()); + assert_eq!(args.storage, StorageBackend::Memory); + assert_eq!(args.lease_seconds, 45); + assert_eq!(args.dashboard_dir, Some(PathBuf::from("dashboard/dist"))); + let _ = std::fs::remove_file(config_path); + } + #[test] fn health_report_includes_controller_and_scheduler_status() { let scheduler = AppScheduler::Memory(InMemoryScheduler::default()); @@ -632,4 +789,11 @@ mod tests { assert_eq!(report.scheduler.backend.as_deref(), Some("memory")); assert!(report.degraded.is_empty()); } + + fn write_config(name: &str, content: &str) -> PathBuf { + let path = + std::env::temp_dir().join(format!("renderacre-{name}-{}.yaml", uuid::Uuid::new_v4())); + std::fs::write(&path, content).expect("config should write"); + path + } } diff --git a/crates/farm-worker/Cargo.toml b/crates/farm-worker/Cargo.toml index 789a059..172bc74 100644 --- a/crates/farm-worker/Cargo.toml +++ b/crates/farm-worker/Cargo.toml @@ -18,7 +18,9 @@ openjd-expr.workspace = true openjd-model.workspace = true openjd-sessions.workspace = true reqwest = { workspace = true, default-features = false, features = ["json", "rustls-tls"] } +serde.workspace = true serde_json.workspace = true +serde_yaml.workspace = true tempfile.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/crates/farm-worker/src/main.rs b/crates/farm-worker/src/main.rs index f92cff0..7fa8d71 100644 --- a/crates/farm-worker/src/main.rs +++ b/crates/farm-worker/src/main.rs @@ -1,10 +1,13 @@ use std::collections::{HashMap, HashSet}; +#[cfg(test)] +use std::ffi::OsString; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::{Duration, SystemTime}; use anyhow::{Context, Result}; -use clap::Parser; +use clap::parser::ValueSource; +use clap::{ArgMatches, CommandFactory, FromArgMatches, Parser}; use farm_core::{ ArtifactKind, LogLevel, OpenJdRuntimeTask, Task, TaskArtifact, TaskComplete, TaskLease, TaskLeaseRenewal, TaskStarted, WorkerCapacity, WorkerId, WorkerInfo, WorkerLogBatch, @@ -15,29 +18,120 @@ use futures_util::stream::{FuturesUnordered, StreamExt}; use openjd_expr::SerializedSymbolTable; use openjd_model::{ModelExtension, ModelProfile, SpecificationRevision, TaskParameterSet}; use openjd_sessions::{ActionState, ActionStatus, Session, SessionConfig, StickyBitPolicy}; +use serde::Deserialize; use tokio::io::{AsyncBufReadExt, AsyncRead, BufReader}; use tokio::process::Command; #[derive(Debug, Parser)] struct Args { + #[arg(long, env = "RFARM_CONFIG")] + config: Option, #[arg( long, env = "RFARM_CONTROLLER", default_value = "http://127.0.0.1:7878" )] controller: String, - #[arg(long)] + #[arg(long, env = "RFARM_WORKER_NAME")] name: Option, - #[arg(long = "label", value_parser = parse_label)] + #[arg( + long = "label", + env = "RFARM_WORKER_LABELS", + value_delimiter = ',', + value_parser = parse_label + )] labels: Vec<(String, String)>, - #[arg(long, default_value_t = 1)] + #[arg(long, env = "RFARM_WORKER_SLOTS", default_value_t = 1)] slots: u32, - #[arg(long, default_value_t = 2)] + #[arg(long, env = "RFARM_WORKER_POLL_SECONDS", default_value_t = 2)] poll_seconds: u64, #[arg(long, env = "RFARM_LEASE_RENEW_SECONDS", default_value_t = 30)] lease_renew_seconds: u64, } +#[derive(Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct WorkerConfig { + controller: Option, + name: Option, + labels: HashMap, + slots: Option, + poll_seconds: Option, + lease_renew_seconds: Option, +} + +impl Args { + fn load() -> Result { + Self::from_matches(Self::command().get_matches()) + } + + #[cfg(test)] + fn load_from(iter: I) -> Result + where + I: IntoIterator, + T: Into + Clone, + { + let matches = Self::command().try_get_matches_from(iter)?; + Self::from_matches(matches) + } + + fn from_matches(matches: ArgMatches) -> Result { + let mut args = Self::from_arg_matches(&matches)?; + let config = WorkerConfig::load(args.config.as_deref())?; + args.apply_config(config, &matches); + Ok(args) + } + + fn apply_config(&mut self, config: WorkerConfig, matches: &ArgMatches) { + if value_from_default(matches, "controller") { + if let Some(controller) = config.controller { + self.controller = controller; + } + } + if matches.value_source("name").is_none() { + if let Some(name) = config.name { + self.name = Some(name); + } + } + if matches.value_source("labels").is_none() && !config.labels.is_empty() { + self.labels = config.labels.into_iter().collect(); + self.labels + .sort_by(|left, right| left.0.cmp(&right.0).then(left.1.cmp(&right.1))); + } + if value_from_default(matches, "slots") { + if let Some(slots) = config.slots { + self.slots = slots; + } + } + if value_from_default(matches, "poll_seconds") { + if let Some(poll_seconds) = config.poll_seconds { + self.poll_seconds = poll_seconds; + } + } + if value_from_default(matches, "lease_renew_seconds") { + if let Some(lease_renew_seconds) = config.lease_renew_seconds { + self.lease_renew_seconds = lease_renew_seconds; + } + } + } +} + +impl WorkerConfig { + fn load(path: Option<&Path>) -> Result { + let Some(path) = path else { + return Ok(Self::default()); + }; + let content = std::fs::read_to_string(path) + .with_context(|| format!("failed to read config file {}", path.display()))?; + serde_yaml::from_str(&content) + .with_context(|| format!("failed to parse config file {}", path.display())) + } +} + +fn value_from_default(matches: &ArgMatches, id: &str) -> bool { + matches.value_source(id) == Some(ValueSource::DefaultValue) +} + #[derive(Clone)] struct WorkerLogSink { client: reqwest::Client, @@ -122,7 +216,7 @@ async fn main() -> Result<()> { ) .init(); - let args = Args::parse(); + let args = Args::load()?; let client = reqwest::Client::new(); let controller = args.controller.trim_end_matches('/').to_string(); let worker = register_worker(&client, &controller, &args).await?; @@ -863,3 +957,86 @@ fn default_worker_name() -> String { .or_else(|_| std::env::var("HOSTNAME")) .unwrap_or_else(|_| "local-worker".to_string()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn config_file_overrides_worker_defaults() { + let config_path = write_config( + "worker", + r#" +controller: http://renderacre-controller:7878 +name: render-node-01 +labels: + app: blender + pool: lighting +slots: 4 +poll_seconds: 5 +lease_renew_seconds: 20 +"#, + ); + + let args = Args::load_from([ + "renderacre-worker", + "--config", + config_path.to_str().unwrap(), + ]) + .expect("config should load"); + + assert_eq!(args.controller, "http://renderacre-controller:7878"); + assert_eq!(args.name.as_deref(), Some("render-node-01")); + assert_eq!( + args.labels, + vec![ + ("app".to_string(), "blender".to_string()), + ("pool".to_string(), "lighting".to_string()) + ] + ); + assert_eq!(args.slots, 4); + assert_eq!(args.poll_seconds, 5); + assert_eq!(args.lease_renew_seconds, 20); + let _ = std::fs::remove_file(config_path); + } + + #[test] + fn command_line_values_override_worker_config() { + let config_path = write_config( + "worker-cli", + r#" +controller: http://renderacre-controller:7878 +name: render-node-01 +labels: + app: blender +slots: 4 +"#, + ); + + let args = Args::load_from([ + "renderacre-worker", + "--config", + config_path.to_str().unwrap(), + "--controller", + "http://127.0.0.1:7878", + "--label", + "app=maya", + "--slots", + "2", + ]) + .expect("config should load"); + + assert_eq!(args.controller, "http://127.0.0.1:7878"); + assert_eq!(args.labels, vec![("app".to_string(), "maya".to_string())]); + assert_eq!(args.slots, 2); + assert_eq!(args.name.as_deref(), Some("render-node-01")); + let _ = std::fs::remove_file(config_path); + } + + fn write_config(name: &str, content: &str) -> PathBuf { + let path = + std::env::temp_dir().join(format!("renderacre-{name}-{}.yaml", uuid::Uuid::new_v4())); + std::fs::write(&path, content).expect("config should write"); + path + } +} diff --git a/deploy/lightweight/controller.yaml b/deploy/lightweight/controller.yaml new file mode 100644 index 0000000..e285e80 --- /dev/null +++ b/deploy/lightweight/controller.yaml @@ -0,0 +1,7 @@ +# Lightweight single-node controller profile. +# Paths are resolved relative to the controller working directory. +bind: 0.0.0.0:7878 +lease_seconds: 120 +storage: sqlite +sqlite_path: ./var/renderacre/renderacre.sqlite3 +dashboard_dir: ./dashboard/dist diff --git a/deploy/lightweight/systemd/renderacre-controller.service b/deploy/lightweight/systemd/renderacre-controller.service new file mode 100644 index 0000000..29cb32c --- /dev/null +++ b/deploy/lightweight/systemd/renderacre-controller.service @@ -0,0 +1,18 @@ +[Unit] +Description=Renderacre Controller +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=renderacre +Group=renderacre +WorkingDirectory=/opt/renderacre +Environment=RUST_LOG=farm_controller=info,tower_http=info +ExecStart=/usr/local/bin/renderacre-controller --config /etc/renderacre/controller.yaml +Restart=on-failure +RestartSec=5 +NoNewPrivileges=true + +[Install] +WantedBy=multi-user.target diff --git a/deploy/lightweight/systemd/renderacre-worker.service b/deploy/lightweight/systemd/renderacre-worker.service new file mode 100644 index 0000000..983f4c7 --- /dev/null +++ b/deploy/lightweight/systemd/renderacre-worker.service @@ -0,0 +1,18 @@ +[Unit] +Description=Renderacre Worker +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=renderacre +Group=renderacre +WorkingDirectory=/var/lib/renderacre-worker +Environment=RUST_LOG=farm_worker=info +ExecStart=/usr/local/bin/renderacre-worker --config /etc/renderacre/worker.yaml +Restart=on-failure +RestartSec=5 +NoNewPrivileges=true + +[Install] +WantedBy=multi-user.target diff --git a/deploy/lightweight/windows/install-controller-service.ps1 b/deploy/lightweight/windows/install-controller-service.ps1 new file mode 100644 index 0000000..e11d73b --- /dev/null +++ b/deploy/lightweight/windows/install-controller-service.ps1 @@ -0,0 +1,29 @@ +param( + [string]$ServiceName = "RenderacreController", + [string]$Executable = "C:\Program Files\Renderacre\renderacre-controller.exe", + [string]$Config = "C:\ProgramData\Renderacre\controller.yaml" +) + +$ErrorActionPreference = "Stop" + +if (-not (Test-Path -LiteralPath $Executable)) { + throw "Controller executable not found: $Executable" +} +if (-not (Test-Path -LiteralPath $Config)) { + throw "Controller config not found: $Config" +} + +$binaryPath = "`"$Executable`" --config `"$Config`"" + +if (Get-Service -Name $ServiceName -ErrorAction SilentlyContinue) { + sc.exe config $ServiceName binPath= $binaryPath | Out-Null +} else { + New-Service ` + -Name $ServiceName ` + -DisplayName "Renderacre Controller" ` + -Description "Renderacre durable controller and dashboard" ` + -BinaryPathName $binaryPath ` + -StartupType Automatic | Out-Null +} + +Write-Host "Installed $ServiceName. Start it with: Start-Service $ServiceName" diff --git a/deploy/lightweight/windows/install-worker-service.ps1 b/deploy/lightweight/windows/install-worker-service.ps1 new file mode 100644 index 0000000..be39ced --- /dev/null +++ b/deploy/lightweight/windows/install-worker-service.ps1 @@ -0,0 +1,29 @@ +param( + [string]$ServiceName = "RenderacreWorker", + [string]$Executable = "C:\Program Files\Renderacre\renderacre-worker.exe", + [string]$Config = "C:\ProgramData\Renderacre\worker.yaml" +) + +$ErrorActionPreference = "Stop" + +if (-not (Test-Path -LiteralPath $Executable)) { + throw "Worker executable not found: $Executable" +} +if (-not (Test-Path -LiteralPath $Config)) { + throw "Worker config not found: $Config" +} + +$binaryPath = "`"$Executable`" --config `"$Config`"" + +if (Get-Service -Name $ServiceName -ErrorAction SilentlyContinue) { + sc.exe config $ServiceName binPath= $binaryPath | Out-Null +} else { + New-Service ` + -Name $ServiceName ` + -DisplayName "Renderacre Worker" ` + -Description "Renderacre render worker" ` + -BinaryPathName $binaryPath ` + -StartupType Automatic | Out-Null +} + +Write-Host "Installed $ServiceName. Start it with: Start-Service $ServiceName" diff --git a/deploy/lightweight/worker.yaml b/deploy/lightweight/worker.yaml new file mode 100644 index 0000000..4c8cc41 --- /dev/null +++ b/deploy/lightweight/worker.yaml @@ -0,0 +1,9 @@ +# Lightweight worker profile. Copy per render node and adjust name, labels, and slots. +controller: http://127.0.0.1:7878 +name: render-node-01 +labels: + pool: default + app: blender +slots: 1 +poll_seconds: 2 +lease_renew_seconds: 30 diff --git a/docs/architecture.md b/docs/architecture.md index 6f8fd96..7520fee 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -149,7 +149,7 @@ the path resolution and byte-serving implementation. ## Dashboard path -`dashboard/` is a Vite React application for queue operations. It reads `/v1/dashboard`, `/v1/jobs`, `/v1/workers`, and `/v1/stats`, then renders a Deadline-style queue table, worker assignment panel, OpenJD step detail, dependency mini graph, attempt history, artifacts, and stdout/stderr tails. During local development Vite proxies `/v1` to the controller; in deployment, serve the built static assets from `dashboard/dist` beside the controller API. +`dashboard/` is a Vite React application for queue operations. It reads `/v1/dashboard`, `/v1/jobs`, `/v1/workers`, and `/v1/stats`, then renders a Deadline-style queue table, worker assignment panel, OpenJD step detail, dependency mini graph, attempt history, artifacts, and stdout/stderr tails. During local development Vite proxies `/v1` to the controller; in deployment, pass `--dashboard-dir dashboard/dist` or `RFARM_DASHBOARD_DIR` to serve the built static assets from the controller, or serve the same directory from an internal web server or reverse proxy next to the controller API. ## Release assets diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..1427315 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,159 @@ +# Renderacre lightweight deployment + +This guide packages the small-studio profile: one controller, SQLite durability, +the built dashboard served by the controller, shared storage for scene/output +paths, and one worker process on each render node. It does not require +Kubernetes, an external database, or a multi-service control plane. + +## Single-node controller + +Build the dashboard once on the controller host: + +```powershell +npm --prefix dashboard ci +npm --prefix dashboard run build +``` + +Then start the durable controller and dashboard with one command: + +```powershell +renderacre-controller --config deploy/lightweight/controller.yaml +``` + +The sample profile stores scheduler state in `./var/renderacre/renderacre.sqlite3` +and serves `./dashboard/dist` from the same HTTP listener as the REST API. In a +source checkout, create the data directory before the first run: + +```powershell +New-Item -ItemType Directory -Force var/renderacre +``` + +Equivalent environment configuration is available when a config file is not +convenient: + +```powershell +$env:RFARM_BIND = "0.0.0.0:7878" +$env:RFARM_STORAGE = "sqlite" +$env:RFARM_SQLITE_PATH = "C:\ProgramData\Renderacre\renderacre.sqlite3" +$env:RFARM_DASHBOARD_DIR = "C:\ProgramData\Renderacre\dashboard" +renderacre-controller +``` + +CLI arguments and environment variables override config-file values. Omitted +values fall back to the built-in defaults. + +## Worker install and registration + +Install the release binaries on every worker host: + +```powershell +powershell -ExecutionPolicy Bypass -File scripts/install.ps1 +``` + +```bash +curl -fsSL https://raw.githubusercontent.com/loonghao/renderacre/main/scripts/install.sh | sh +``` + +Windows workers can use either the installed binary directory or an explicit +path to `renderacre-worker.exe`. macOS and Linux workers can place +`renderacre-worker` in `/usr/local/bin` or another service account path. + +Register a worker from a config file: + +```powershell +renderacre-worker --config deploy/lightweight/worker.yaml +``` + +Or use environment variables: + +```powershell +$env:RFARM_CONTROLLER = "http://controller-host:7878" +$env:RFARM_WORKER_NAME = $env:COMPUTERNAME +$env:RFARM_WORKER_LABELS = "pool=lighting,app=blender" +$env:RFARM_WORKER_SLOTS = "4" +renderacre-worker +``` + +Labels describe scheduling capabilities. Use `pool`, `app`, operating-system, +GPU, or DCC-specific labels that match submitted task requirements. + +## Service examples + +Systemd examples live in `deploy/lightweight/systemd/`. +Create a `renderacre` service user first, then install the config and service +files: + +```bash +sudo install -d -o renderacre -g renderacre /etc/renderacre /var/lib/renderacre /opt/renderacre +sudo install -m 0644 deploy/lightweight/controller.yaml /etc/renderacre/controller.yaml +sudo install -m 0644 deploy/lightweight/worker.yaml /etc/renderacre/worker.yaml +sudo install -m 0644 deploy/lightweight/systemd/renderacre-controller.service /etc/systemd/system/ +sudo install -m 0644 deploy/lightweight/systemd/renderacre-worker.service /etc/systemd/system/ +``` + +Edit `/etc/renderacre/controller.yaml` so `sqlite_path` points at +`/var/lib/renderacre/renderacre.sqlite3` and `dashboard_dir` points at the +deployed dashboard assets, for example `/opt/renderacre/dashboard/dist`. Then +enable the services: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable --now renderacre-controller +sudo systemctl enable --now renderacre-worker +``` + +Windows service examples live in `deploy/lightweight/windows/`. Run PowerShell as +Administrator after copying the binaries, config files, and dashboard assets to +their final paths: + +```powershell +powershell -ExecutionPolicy Bypass -File deploy/lightweight/windows/install-controller-service.ps1 +powershell -ExecutionPolicy Bypass -File deploy/lightweight/windows/install-worker-service.ps1 +Start-Service RenderacreController +Start-Service RenderacreWorker +``` + +For service-managed Windows installs, prefer absolute paths in +`C:\ProgramData\Renderacre\controller.yaml` because the Windows service working +directory is manager-dependent. + +## Backup and upgrades + +For the default SQLite profile, back up the controller database before upgrades. +The safest path is to stop the controller, copy the SQLite file, upgrade the +binaries, then start the controller again: + +```powershell +Stop-Service RenderacreController +Copy-Item C:\ProgramData\Renderacre\renderacre.sqlite3 C:\Backups\renderacre.sqlite3 +Start-Service RenderacreController +``` + +On Linux: + +```bash +sudo systemctl stop renderacre-controller +sudo cp /var/lib/renderacre/renderacre.sqlite3 /var/backups/renderacre.sqlite3 +sudo systemctl start renderacre-controller +``` + +Upgrade controller binaries before worker binaries when a release changes API +behavior. Keep the previous controller and worker binaries until the dashboard +loads, `/readyz` returns `status: "ok"`, and workers register successfully. + +## Growing the farm + +Keep the lightweight profile until SQLite backup windows, controller CPU, or +network placement becomes the bottleneck. The next steps are additive: + +- Put the controller behind a reverse proxy that owns TLS, authentication, and + network policy. +- Move dashboard assets to the proxy or static web tier if controller-local + serving is no longer desirable. +- Graduate the storage boundary from SQLite to a future Postgres or managed + backend while preserving submitter and worker contracts. +- Move large logs and artifacts to object storage behind the controller artifact + APIs. + +The stable and experimental extension boundaries are described in +[extension-contracts.md](extension-contracts.md).