Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,22 @@ Workers advertise matching capabilities with labels such as
also register default OpenJD capabilities for OS family, CPU architecture, and
slot count, so OpenJD `hostRequirements` work out of the box for common cases.

Operators can define shared limits for scarce licenses or farm-wide resources,
then tasks can declare the named limits they need:

```text
POST /v1/limits
GET /v1/limits
```

```json
{
"name": "maya-render",
"command": { "executable": "mayapy", "args": ["render.py"] },
"limits": ["maya"]
}
```

Common queue operations are available as stable action endpoints:

```text
Expand Down
36 changes: 34 additions & 2 deletions crates/farm-controller/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@ use axum::{Json, Router};
use clap::{Parser, ValueEnum};
use farm_core::{
DashboardSnapshot, FarmError, FarmLogEntry, FarmStats, InMemoryScheduler, Job, JobId,
JobPriorityUpdate, JobSubmit, SchedulerConfig, SqliteScheduler, Task, TaskComplete, TaskId,
TaskLease, TaskLeaseRenewal, TaskStarted, WorkerId, WorkerInfo, WorkerLogBatch, WorkerRegister,
JobPriorityUpdate, JobSubmit, ResourceLimitDefinition, ResourceLimitSnapshot, SchedulerConfig,
SqliteScheduler, Task, TaskComplete, TaskId, TaskLease, TaskLeaseRenewal, TaskStarted,
WorkerId, WorkerInfo, WorkerLogBatch, WorkerRegister,
};
use serde_json::json;
use tower_http::cors::CorsLayer;
Expand Down Expand Up @@ -102,6 +103,23 @@ impl AppScheduler {
}
}

fn define_limit(
&self,
definition: ResourceLimitDefinition,
) -> Result<ResourceLimitSnapshot, FarmError> {
match self {
Self::Memory(scheduler) => scheduler.define_limit(definition),
Self::Sqlite(scheduler) => scheduler.define_limit(definition),
}
}

fn list_limits(&self) -> Result<Vec<ResourceLimitSnapshot>, FarmError> {
match self {
Self::Memory(scheduler) => scheduler.list_limits(),
Self::Sqlite(scheduler) => scheduler.list_limits(),
}
}

fn pause_job(&self, job_id: JobId) -> Result<Job, FarmError> {
match self {
Self::Memory(scheduler) => scheduler.pause_job(job_id),
Expand Down Expand Up @@ -234,6 +252,7 @@ fn app(scheduler: AppScheduler) -> Router {
.route("/v1/dashboard", get(get_dashboard))
.route("/v1/logs", get(list_logs))
.route("/v1/stats", get(get_stats))
.route("/v1/limits", get(list_limits).post(define_limit))
.route("/v1/jobs", get(list_jobs).post(submit_job))
.route("/v1/jobs/{job_id}", get(get_job))
.route("/v1/jobs/{job_id}/pause", post(pause_job))
Expand Down Expand Up @@ -337,6 +356,19 @@ async fn list_logs(
Ok(Json(scheduler.list_logs()?))
}

async fn define_limit(
State(scheduler): State<AppScheduler>,
Json(definition): Json<ResourceLimitDefinition>,
) -> Result<Json<ResourceLimitSnapshot>, ApiError> {
Ok(Json(scheduler.define_limit(definition)?))
}

async fn list_limits(
State(scheduler): State<AppScheduler>,
) -> Result<Json<Vec<ResourceLimitSnapshot>>, ApiError> {
Ok(Json(scheduler.list_limits()?))
}

async fn register_worker(
State(scheduler): State<AppScheduler>,
Json(registration): Json<WorkerRegister>,
Expand Down
20 changes: 20 additions & 0 deletions crates/farm-core/src/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ pub struct TaskSubmit {
#[serde(default)]
pub requirements: TaskRequirements,
#[serde(default)]
pub limits: Vec<String>,
#[serde(default)]
pub max_retries: Option<u32>,
#[serde(default)]
pub artifact_paths: Vec<PathBuf>,
Expand Down Expand Up @@ -431,6 +433,8 @@ pub struct DashboardSnapshot {
pub jobs: Vec<Job>,
pub workers: Vec<WorkerInfo>,
pub logs: Vec<FarmLogEntry>,
#[serde(default)]
pub limits: Vec<ResourceLimitSnapshot>,
}

#[derive(Debug, Clone, Default, Serialize, Deserialize)]
Expand Down Expand Up @@ -468,6 +472,8 @@ pub struct Task {
pub dependencies: Vec<TaskId>,
#[serde(default)]
pub requirements: TaskRequirements,
#[serde(default)]
pub limits: Vec<String>,
pub attempts: u32,
pub max_retries: u32,
pub lease: Option<TaskLeaseInfo>,
Expand Down Expand Up @@ -610,6 +616,20 @@ pub enum LogSource {
Task,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ResourceLimitDefinition {
pub name: String,
pub max_count: u32,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ResourceLimitSnapshot {
pub name: String,
pub max_count: u32,
pub used: u32,
pub available: u32,
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum WorkerState {
Expand Down
1 change: 1 addition & 0 deletions crates/farm-core/src/openjd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,7 @@ fn expand_openjd_tasks(
.as_ref()
.map(openjd_host_requirements)
.unwrap_or_default(),
limits: Vec::new(),
max_retries: None,
artifact_paths: infer_artifact_paths(&job_parameters),
openjd: Some(openjd),
Expand Down
Loading
Loading