From 3502f0ff85395556954e3e0d4462215161643d05 Mon Sep 17 00:00:00 2001 From: Jean Schmidt Date: Wed, 24 Jun 2026 15:54:23 -0700 Subject: [PATCH 1/5] Update [ghstack-poisoned] --- osdc/clusters.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/osdc/clusters.yaml b/osdc/clusters.yaml index b69b3818..d8ef4a7d 100644 --- a/osdc/clusters.yaml +++ b/osdc/clusters.yaml @@ -246,6 +246,7 @@ clusters: - karpenter - arc - nodepools + - bin-pack-scheduler - arc-runners - keda - buildkit From 8caae5548babd98e43b318ef176973acba8b2f19 Mon Sep 17 00:00:00 2001 From: Jean Schmidt Date: Wed, 24 Jun 2026 15:54:23 -0700 Subject: [PATCH 2/5] Update (base update) [ghstack-poisoned] --- .../scripts/python/generate_runners.py | 49 ++- .../scripts/python/test_generate_runners.py | 281 ++++++++++++++++++ .../arc-runners/templates/runner.yaml.tpl | 7 +- .../tests/smoke/test_scheduler_name.py | 178 +++++++++++ .../bin-pack-scheduler/kubernetes/config.yaml | 24 ++ .../kubernetes/deployment.yaml | 73 +++++ .../kubernetes/kustomization.yaml | 18 ++ .../bin-pack-scheduler/kubernetes/rbac.yaml | 91 ++++++ .../scripts/python/test_image_version.py | 34 +++ .../tests/smoke/conftest.py | 1 + .../tests/smoke/test_bin_pack_scheduler.py | 26 ++ 11 files changed, 779 insertions(+), 3 deletions(-) create mode 100644 osdc/modules/arc-runners/tests/smoke/test_scheduler_name.py create mode 100644 osdc/modules/bin-pack-scheduler/kubernetes/config.yaml create mode 100644 osdc/modules/bin-pack-scheduler/kubernetes/deployment.yaml create mode 100644 osdc/modules/bin-pack-scheduler/kubernetes/kustomization.yaml create mode 100644 osdc/modules/bin-pack-scheduler/kubernetes/rbac.yaml create mode 100644 osdc/modules/bin-pack-scheduler/scripts/python/test_image_version.py create mode 100644 osdc/modules/bin-pack-scheduler/tests/smoke/conftest.py create mode 100644 osdc/modules/bin-pack-scheduler/tests/smoke/test_bin_pack_scheduler.py diff --git a/osdc/modules/arc-runners/scripts/python/generate_runners.py b/osdc/modules/arc-runners/scripts/python/generate_runners.py index 85b31c61..d883667a 100755 --- a/osdc/modules/arc-runners/scripts/python/generate_runners.py +++ b/osdc/modules/arc-runners/scripts/python/generate_runners.py @@ -173,14 +173,28 @@ def resolve_max_runners(value, def_file, cluster_id): return value -def generate_runner(def_file, template_content, cluster_config, output_dir, module_name, pypi_cache_enabled=True): +def generate_runner( + def_file, + template_content, + cluster_config, + output_dir, + module_name, + pypi_cache_enabled=True, + available_modules=None, +): """Generate a single runner config from its definition. pypi_cache_enabled controls whether the `# BEGIN_PYPI_CACHE` / `# END_PYPI_CACHE` block in the template is preserved (True) or stripped (False). Strip when the cluster does not deploy the pypi-cache module — the env vars would otherwise point at a Service that doesn't exist on this cluster. + + available_modules is the set of modules the cluster deploys. A resolved + scheduler_name that does not name an available module is silently dropped, + so workflow pods don't get stamped with a schedulerName that has no + scheduler running (they would pend forever). """ + available_modules = available_modules or set() with open(def_file) as f: data = yaml.safe_load(f) @@ -335,6 +349,26 @@ def generate_runner(def_file, template_content, cluster_config, output_dir, modu # Optional maxRunners line — only emitted when max_runners is set in the def max_runners_line = f"maxRunners: {max_runners}" if max_runners is not None else "" + # Optional schedulerName for workflow pods (per-def scheduler_name). + # Empty = default scheduler. The same value feeds two places so the real + # workflow pod and its capacity placeholder (ph-w-*) agree on the scheduler: + # {{SCHEDULER_NAME_LINE}} -> schedulerName on the workflow pod spec + # {{SCHEDULER_NAME}} -> CAPACITY_AWARE_WORKFLOW_SCHEDULER_NAME on the + # listener, which the fork stamps onto ph-w-*. + # If they diverged, the placeholder would reserve a slot the real pod can't claim. + scheduler_name = runner.get("scheduler_name", "") + # Cluster-wide default: a def without its own scheduler_name inherits + # arc-runners.scheduler_name from clusters.yaml, so a cluster can route all + # workflow pods to a secondary scheduler from one place (per-def value wins). + if not scheduler_name: + scheduler_name = cluster_config.get("scheduler_name", "") + # Convention: module dir name == scheduler name. If the cluster does not + # deploy a module with this name, silently drop — otherwise the workflow pod + # would be stamped with a schedulerName that has no scheduler running. + if scheduler_name and scheduler_name not in available_modules: + scheduler_name = "" + scheduler_name_line = f" schedulerName: {scheduler_name}" if scheduler_name else "" + # Replace all template placeholders output_content = template_content output_content = strip_conditional_block(output_content, "PYPI_CACHE", keep=pypi_cache_enabled) @@ -364,6 +398,8 @@ def generate_runner(def_file, template_content, cluster_config, output_dir, modu "{{PROACTIVE_CAPACITY}}": str(proactive_capacity), "{{MAX_BURST_CAPACITY}}": str(max_burst_capacity), "{{HUD_FAILURE_BASE_CAPACITY}}": str(hud_failure_base_capacity), + "{{SCHEDULER_NAME_LINE}}": scheduler_name_line, + "{{SCHEDULER_NAME}}": scheduler_name, } for placeholder, value in replacements.items(): @@ -498,10 +534,19 @@ def main(): # vars from the workflow pod template so jobs don't try to reach a Service # that doesn't exist on this cluster. pypi_cache_enabled = "pypi-cache" in (cluster_cfg.get("modules") or []) + available_modules = set(cluster_cfg.get("modules") or []) count = 0 for def_file in def_files: - if generate_runner(def_file, template_content, cluster_config, output_dir, module_name, pypi_cache_enabled): + if generate_runner( + def_file, + template_content, + cluster_config, + output_dir, + module_name, + pypi_cache_enabled, + available_modules, + ): count += 1 print() diff --git a/osdc/modules/arc-runners/scripts/python/test_generate_runners.py b/osdc/modules/arc-runners/scripts/python/test_generate_runners.py index 11da8918..68891d1c 100644 --- a/osdc/modules/arc-runners/scripts/python/test_generate_runners.py +++ b/osdc/modules/arc-runners/scripts/python/test_generate_runners.py @@ -193,6 +193,7 @@ def make_def_file( max_burst_capacity=None, hud_failure_base_capacity=None, node_fleet=None, + scheduler_name=None, ): """Write a runner def YAML and return the path. @@ -221,6 +222,8 @@ def make_def_file( runner["hud_failure_base_capacity"] = hud_failure_base_capacity if node_fleet is not None: runner["node_fleet"] = node_fleet + if scheduler_name is not None: + runner["scheduler_name"] = scheduler_name content = {"runner": runner} p = tmp_path / f"{name}.yaml" p.write_text(yaml.dump(content, default_flow_style=False)) @@ -1858,6 +1861,19 @@ def _render_real(real_template, tmp_path, variant): return helm, configmap, workflow_pod +def _listener_env_value(helm, name): + """Return the value of a named env var on the listener container. + + Returns None when the env var is absent so callers can distinguish + "missing" from "present but empty". + """ + containers = helm["listenerTemplate"]["spec"]["containers"] + for entry in containers[0].get("env", []): + if entry.get("name") == name: + return entry.get("value") + return None + + class TestRealTemplate: @pytest.mark.parametrize("variant", RUNNER_VARIANTS) def test_runner_pod_uses_arc_runner_priority_class(self, real_template, tmp_path, variant): @@ -1871,6 +1887,271 @@ def test_runner_pod_pinned_to_c7i_runner_fleet(self, real_template, tmp_path, va helm, _, _ = _render_real(real_template, tmp_path, variant) assert helm["template"]["spec"]["nodeSelector"]["node-fleet"] == "c7i-runner" + def test_workflow_pod_scheduler_name_from_def(self, real_template, tmp_path): + """Workflow pod gets schedulerName when the runner def sets scheduler_name.""" + def_file = make_def_file( + tmp_path=tmp_path, + name="packed-runner", + instance_type="r7a.48xlarge", + vcpu=8, + memory=64, + gpu=0, + disk_size=200, + scheduler_name="bin-pack-scheduler", + ) + output_dir = tmp_path / "out" + output_dir.mkdir() + cluster_config = { + "github_config_url": "https://github.com/test-org", + "github_secret_name": "gh-secret", + "runner_name_prefix": "real-", + } + assert ( + generate_runner( + def_file, + real_template, + cluster_config, + output_dir, + "arc-runners", + available_modules={"bin-pack-scheduler"}, + ) + is True + ) + docs = list(yaml.safe_load_all((output_dir / "packed-runner.yaml").read_text())) + helm = docs[0] + workflow_pod = yaml.safe_load(docs[1]["data"]["job-pod.yaml"]) + assert workflow_pod["spec"]["schedulerName"] == "bin-pack-scheduler" + # The capacity placeholder (ph-w-*) must use the SAME scheduler so it + # reserves a slot the real workflow pod can actually claim. + assert _listener_env_value(helm, "CAPACITY_AWARE_WORKFLOW_SCHEDULER_NAME") == "bin-pack-scheduler" + + def test_workflow_pod_scheduler_name_from_cluster_default(self, real_template, tmp_path): + """A def without its own scheduler_name inherits arc-runners.scheduler_name.""" + def_file = make_def_file( + tmp_path=tmp_path, + name="packed-runner", + instance_type="r7a.48xlarge", + vcpu=8, + memory=64, + gpu=0, + disk_size=200, + ) + output_dir = tmp_path / "out" + output_dir.mkdir() + cluster_config = { + "github_config_url": "https://github.com/test-org", + "github_secret_name": "gh-secret", + "runner_name_prefix": "real-", + "scheduler_name": "bin-pack-scheduler", + } + assert ( + generate_runner( + def_file, + real_template, + cluster_config, + output_dir, + "arc-runners", + available_modules={"bin-pack-scheduler"}, + ) + is True + ) + docs = list(yaml.safe_load_all((output_dir / "packed-runner.yaml").read_text())) + helm = docs[0] + workflow_pod = yaml.safe_load(docs[1]["data"]["job-pod.yaml"]) + assert workflow_pod["spec"]["schedulerName"] == "bin-pack-scheduler" + assert _listener_env_value(helm, "CAPACITY_AWARE_WORKFLOW_SCHEDULER_NAME") == "bin-pack-scheduler" + + def test_def_scheduler_name_overrides_cluster_default(self, real_template, tmp_path): + """A def's own scheduler_name wins over the cluster default.""" + def_file = make_def_file( + tmp_path=tmp_path, + name="packed-runner", + instance_type="r7a.48xlarge", + vcpu=8, + memory=64, + gpu=0, + disk_size=200, + scheduler_name="def-sched", + ) + output_dir = tmp_path / "out" + output_dir.mkdir() + cluster_config = { + "github_config_url": "https://github.com/test-org", + "github_secret_name": "gh-secret", + "runner_name_prefix": "real-", + "scheduler_name": "cluster-sched", + } + assert ( + generate_runner( + def_file, + real_template, + cluster_config, + output_dir, + "arc-runners", + available_modules={"def-sched", "cluster-sched"}, + ) + is True + ) + docs = list(yaml.safe_load_all((output_dir / "packed-runner.yaml").read_text())) + helm = docs[0] + workflow_pod = yaml.safe_load(docs[1]["data"]["job-pod.yaml"]) + assert workflow_pod["spec"]["schedulerName"] == "def-sched" + assert _listener_env_value(helm, "CAPACITY_AWARE_WORKFLOW_SCHEDULER_NAME") == "def-sched" + + def test_workflow_pod_scheduler_name_dropped_when_module_absent_cluster_default(self, real_template, tmp_path): + """Cluster default scheduler_name is dropped when the module is not deployed.""" + def_file = make_def_file( + tmp_path=tmp_path, + name="packed-runner", + instance_type="r7a.48xlarge", + vcpu=8, + memory=64, + gpu=0, + disk_size=200, + ) + output_dir = tmp_path / "out" + output_dir.mkdir() + cluster_config = { + "github_config_url": "https://github.com/test-org", + "github_secret_name": "gh-secret", + "runner_name_prefix": "real-", + "scheduler_name": "bin-pack-scheduler", + } + assert ( + generate_runner( + def_file, + real_template, + cluster_config, + output_dir, + "arc-runners", + available_modules=set(), + ) + is True + ) + docs = list(yaml.safe_load_all((output_dir / "packed-runner.yaml").read_text())) + helm = docs[0] + workflow_pod = yaml.safe_load(docs[1]["data"]["job-pod.yaml"]) + assert "schedulerName" not in workflow_pod["spec"] + assert _listener_env_value(helm, "CAPACITY_AWARE_WORKFLOW_SCHEDULER_NAME") == "" + + def test_workflow_pod_scheduler_name_dropped_when_module_absent_per_def(self, real_template, tmp_path): + """Per-def scheduler_name is dropped when the module is not deployed.""" + def_file = make_def_file( + tmp_path=tmp_path, + name="packed-runner", + instance_type="r7a.48xlarge", + vcpu=8, + memory=64, + gpu=0, + disk_size=200, + scheduler_name="bin-pack-scheduler", + ) + output_dir = tmp_path / "out" + output_dir.mkdir() + cluster_config = { + "github_config_url": "https://github.com/test-org", + "github_secret_name": "gh-secret", + "runner_name_prefix": "real-", + } + assert ( + generate_runner( + def_file, + real_template, + cluster_config, + output_dir, + "arc-runners", + available_modules=set(), + ) + is True + ) + docs = list(yaml.safe_load_all((output_dir / "packed-runner.yaml").read_text())) + helm = docs[0] + workflow_pod = yaml.safe_load(docs[1]["data"]["job-pod.yaml"]) + assert "schedulerName" not in workflow_pod["spec"] + assert _listener_env_value(helm, "CAPACITY_AWARE_WORKFLOW_SCHEDULER_NAME") == "" + + def test_workflow_pod_scheduler_name_enabled_when_module_present(self, real_template, tmp_path): + """Cluster default scheduler_name is stamped when the module is in the cluster modules list.""" + def_file = make_def_file( + tmp_path=tmp_path, + name="packed-runner", + instance_type="r7a.48xlarge", + vcpu=8, + memory=64, + gpu=0, + disk_size=200, + ) + output_dir = tmp_path / "out" + output_dir.mkdir() + cluster_config = { + "github_config_url": "https://github.com/test-org", + "github_secret_name": "gh-secret", + "runner_name_prefix": "real-", + "scheduler_name": "bin-pack-scheduler", + } + assert ( + generate_runner( + def_file, + real_template, + cluster_config, + output_dir, + "arc-runners", + available_modules={"bin-pack-scheduler", "pypi-cache"}, + ) + is True + ) + docs = list(yaml.safe_load_all((output_dir / "packed-runner.yaml").read_text())) + helm = docs[0] + workflow_pod = yaml.safe_load(docs[1]["data"]["job-pod.yaml"]) + assert workflow_pod["spec"]["schedulerName"] == "bin-pack-scheduler" + assert _listener_env_value(helm, "CAPACITY_AWARE_WORKFLOW_SCHEDULER_NAME") == "bin-pack-scheduler" + + def test_workflow_pod_per_def_scheduler_dropped_no_fallback_to_cluster_default(self, real_template, tmp_path): + """Per-def scheduler_name wins resolution; if its module is absent it is dropped, with no fallback to the cluster default even when that one's module IS deployed.""" + def_file = make_def_file( + tmp_path=tmp_path, + name="packed-runner", + instance_type="r7a.48xlarge", + vcpu=8, + memory=64, + gpu=0, + disk_size=200, + scheduler_name="def-sched", + ) + output_dir = tmp_path / "out" + output_dir.mkdir() + cluster_config = { + "github_config_url": "https://github.com/test-org", + "github_secret_name": "gh-secret", + "runner_name_prefix": "real-", + "scheduler_name": "cluster-sched", + } + assert ( + generate_runner( + def_file, + real_template, + cluster_config, + output_dir, + "arc-runners", + available_modules={"cluster-sched"}, + ) + is True + ) + docs = list(yaml.safe_load_all((output_dir / "packed-runner.yaml").read_text())) + helm = docs[0] + workflow_pod = yaml.safe_load(docs[1]["data"]["job-pod.yaml"]) + assert "schedulerName" not in workflow_pod["spec"] + assert _listener_env_value(helm, "CAPACITY_AWARE_WORKFLOW_SCHEDULER_NAME") == "" + + @pytest.mark.parametrize("variant", RUNNER_VARIANTS) + def test_workflow_pod_no_scheduler_name_by_default(self, real_template, tmp_path, variant): + """Workflow pod must NOT have schedulerName when the runner def omits scheduler_name.""" + helm, _, workflow_pod = _render_real(real_template, tmp_path, variant) + assert "schedulerName" not in workflow_pod["spec"] + # Placeholder scheduler env must be present-but-empty (the fork treats + # empty as default-scheduler), keeping it in sync with the workflow pod. + assert _listener_env_value(helm, "CAPACITY_AWARE_WORKFLOW_SCHEDULER_NAME") == "" + @pytest.mark.parametrize("variant", RUNNER_VARIANTS) def test_runner_pod_tolerates_c7i_runner_node_fleet(self, real_template, tmp_path, variant): """Runner pod tolerations must include node-fleet=c7i-runner.""" diff --git a/osdc/modules/arc-runners/templates/runner.yaml.tpl b/osdc/modules/arc-runners/templates/runner.yaml.tpl index 878747f0..56618eda 100644 --- a/osdc/modules/arc-runners/templates/runner.yaml.tpl +++ b/osdc/modules/arc-runners/templates/runner.yaml.tpl @@ -179,6 +179,11 @@ listenerTemplate: value: "{{GPU_COUNT}}" - name: CAPACITY_AWARE_WORKFLOW_DISK value: "{{DISK_SIZE}}" + # Stamp the workflow placeholder (ph-w-*) with the same scheduler the + # real workflow pod uses (see {{SCHEDULER_NAME_LINE}} below) so + # packing/reservation stay consistent. Empty = default scheduler. + - name: CAPACITY_AWARE_WORKFLOW_SCHEDULER_NAME + value: "{{SCHEDULER_NAME}}" # Must match the runner container resources below (requests/limits) - name: CAPACITY_AWARE_RUNNER_CPU value: "750m" @@ -383,7 +388,7 @@ data: # Priority 20 — preempts placeholder-workflow (10) so workflow pods # can claim the capacity reserved by the placeholders they replace. priorityClassName: arc-workflow - +{{SCHEDULER_NAME_LINE}} # Prefer scheduling job pods on same node fleet as runner. # Tolerations enforce node-fleet constraints (every NodePool taints # with node-fleet=:NoSchedule), so nodeSelector is not needed. diff --git a/osdc/modules/arc-runners/tests/smoke/test_scheduler_name.py b/osdc/modules/arc-runners/tests/smoke/test_scheduler_name.py new file mode 100644 index 00000000..a81ded9f --- /dev/null +++ b/osdc/modules/arc-runners/tests/smoke/test_scheduler_name.py @@ -0,0 +1,178 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml +from helpers import filter_pods, run_kubectl +from runner_defs import ( + arc_runners_module_names, + def_for_listener_pod, + load_defs_by_name, + load_runner_defs, +) + +pytestmark = [pytest.mark.live] + +NAMESPACE = "arc-runners" +LISTENER_NAMESPACE = "arc-systems" +LISTENER_LABELS = {"app.kubernetes.io/component": "runner-scale-set-listener"} +LISTENER_CONTAINER_NAME = "listener" +MODULE_LABEL_KEY = "osdc.io/module" +SCHEDULER_MODULE = "bin-pack-scheduler" +SCHEDULER_ENV_VAR = "CAPACITY_AWARE_WORKFLOW_SCHEDULER_NAME" + + +def _normalize_name(name: str) -> str: + return name.replace(".", "-").replace("_", "-") + + +def _effective_scheduler_name(runner_def: dict, cluster_default: str) -> str: + own = runner_def.get("scheduler_name", "") + return own if own else cluster_default + + +def _workflow_pod_from_cm_data(data: dict) -> dict | None: + raw = data.get("job-pod.yaml", "") + if not raw: + return None + parsed = yaml.safe_load(raw) + return parsed if isinstance(parsed, dict) else None + + +def _skip_unless_scheduler_configured( + enabled_modules: list[str], + cluster_default_scheduler: str, +) -> None: + if "arc-runners" not in enabled_modules: + pytest.skip("arc-runners module not enabled") + if SCHEDULER_MODULE not in enabled_modules: + pytest.skip(f"{SCHEDULER_MODULE} module not enabled") + if not cluster_default_scheduler: + pytest.skip("arc-runners.scheduler_name not configured for this cluster") + + +class TestSchedulerName: + def test_generated_workflow_pod_has_scheduler_name( + self, + upstream_dir: Path, + enabled_modules: list[str], + resolve_config, + ) -> None: + cluster_default = resolve_config("arc-runners.scheduler_name", "") + _skip_unless_scheduler_configured(enabled_modules, cluster_default) + + enabled_arc = arc_runners_module_names(upstream_dir) & set(enabled_modules) + defs_by_name = load_defs_by_name(upstream_dir, enabled_arc) + + modules_dir = upstream_dir / "modules" + generated_files = [] + for module in sorted(enabled_arc): + generated_files.extend(sorted((modules_dir / module / "generated").glob("*.yaml"))) + assert generated_files, f"No generated YAMLs found for arc-runners modules: {sorted(enabled_arc)}" + + problems: list[str] = [] + for yaml_file in generated_files: + def_name = yaml_file.stem + runner = defs_by_name.get(def_name) + if runner is None: + problems.append(f"{yaml_file}: no runner def matches stem {def_name!r}") + continue + expected = _effective_scheduler_name(runner, cluster_default) + docs = list(yaml.safe_load_all(yaml_file.read_text())) + cm_doc = next( + (d for d in docs if isinstance(d, dict) and d.get("kind") == "ConfigMap"), + None, + ) + if cm_doc is None: + problems.append(f"{yaml_file}: no ConfigMap document in generated YAML") + continue + pod = _workflow_pod_from_cm_data(cm_doc.get("data", {}) or {}) + if pod is None: + problems.append(f"{yaml_file}: ConfigMap has no parseable job-pod.yaml") + continue + got = pod.get("spec", {}).get("schedulerName", "") + if got != expected: + problems.append(f"{def_name}: generated schedulerName={got!r}, expected {expected!r}") + + assert not problems, "Generated workflow pod schedulerName mismatches:\n" + "\n".join(problems) + + def test_deployed_hook_configmap_matches_generated( + self, + upstream_dir: Path, + enabled_modules: list[str], + resolve_config, + ) -> None: + cluster_default = resolve_config("arc-runners.scheduler_name", "") + _skip_unless_scheduler_configured(enabled_modules, cluster_default) + + enabled_arc = arc_runners_module_names(upstream_dir) & set(enabled_modules) + defs = load_runner_defs(upstream_dir, enabled_arc) + + result = run_kubectl(["get", "configmaps", "-l", MODULE_LABEL_KEY, "-o", "json"], namespace=NAMESPACE) + cms_by_name = { + item["metadata"]["name"]: item + for item in result.get("items", []) + if item.get("metadata", {}).get("labels", {}).get(MODULE_LABEL_KEY) in enabled_arc + } + + problems: list[str] = [] + for runner in defs: + cm_name = f"arc-runner-hook-{_normalize_name(runner['name'])}" + cm = cms_by_name.get(cm_name) + if cm is None: + continue + expected = _effective_scheduler_name(runner, cluster_default) + pod = _workflow_pod_from_cm_data(cm.get("data", {}) or {}) + if pod is None: + problems.append(f"{cm_name}: no parseable job-pod.yaml in deployed ConfigMap") + continue + got = pod.get("spec", {}).get("schedulerName", "") + if got != expected: + problems.append(f"{cm_name}: deployed schedulerName={got!r}, expected {expected!r}") + + assert not problems, "Deployed hook ConfigMap schedulerName mismatches:\n" + "\n".join(problems) + + def test_listener_pods_carry_scheduler_env_var( + self, + all_pods: dict, + upstream_dir: Path, + enabled_modules: list[str], + resolve_config, + ) -> None: + cluster_default = resolve_config("arc-runners.scheduler_name", "") + _skip_unless_scheduler_configured(enabled_modules, cluster_default) + + enabled_arc = arc_runners_module_names(upstream_dir) & set(enabled_modules) + defs_by_name = load_defs_by_name(upstream_dir, enabled_arc) + runner_name_prefix = resolve_config("arc-runners.runner_name_prefix", "") + + listener_pods = filter_pods(all_pods, namespace=LISTENER_NAMESPACE, labels=LISTENER_LABELS) + assert len(listener_pods) >= 1, ( + f"No listener pods found in '{LISTENER_NAMESPACE}' with labels {LISTENER_LABELS}" + ) + + problems: list[str] = [] + for pod in listener_pods: + pod_name = pod["metadata"]["name"] + containers = pod.get("spec", {}).get("containers", []) + listener = next((c for c in containers if c.get("name") == LISTENER_CONTAINER_NAME), None) + if listener is None: + problems.append(f"{pod_name}: no '{LISTENER_CONTAINER_NAME}' container") + continue + + def_name, runner = def_for_listener_pod(pod, defs_by_name, runner_name_prefix) + if def_name is None or runner is None: + continue + + expected = _effective_scheduler_name(runner, cluster_default) + env_by_name = {e["name"]: e for e in listener.get("env", []) or []} + entry = env_by_name.get(SCHEDULER_ENV_VAR) + if entry is None: + problems.append(f"{pod_name} (def={def_name}): missing env var {SCHEDULER_ENV_VAR!r}") + continue + got = entry.get("value", "") or "" + if got != expected: + problems.append(f"{pod_name} (def={def_name}): {SCHEDULER_ENV_VAR}={got!r}, expected {expected!r}") + + assert not problems, f"Listener {SCHEDULER_ENV_VAR} mismatches:\n" + "\n".join(problems) diff --git a/osdc/modules/bin-pack-scheduler/kubernetes/config.yaml b/osdc/modules/bin-pack-scheduler/kubernetes/config.yaml new file mode 100644 index 00000000..dc5e2b8a --- /dev/null +++ b/osdc/modules/bin-pack-scheduler/kubernetes/config.yaml @@ -0,0 +1,24 @@ +# KubeSchedulerConfiguration for the bin-pack-scheduler secondary scheduler. +# Stock upstream kube-scheduler; only NodeResourcesFit scoring is overridden to +# MostAllocated (pack onto the fullest node). Every other plugin/default is +# inherited. EKS's managed default scheduler can't be reconfigured, hence a +# second scheduler that pods opt into via spec.schedulerName. +apiVersion: kubescheduler.config.k8s.io/v1 +kind: KubeSchedulerConfiguration +leaderElection: + leaderElect: true + # MUST differ from the managed scheduler's lease ("kube-scheduler"). + resourceName: bin-pack-scheduler + resourceNamespace: kube-system +profiles: + - schedulerName: bin-pack-scheduler + pluginConfig: + - name: NodeResourcesFit + args: + scoringStrategy: + # cpu/memory weights match the upstream default, so `type` is the + # only delta from the managed default-scheduler. + type: MostAllocated + resources: + - {name: cpu, weight: 1} + - {name: memory, weight: 1} diff --git a/osdc/modules/bin-pack-scheduler/kubernetes/deployment.yaml b/osdc/modules/bin-pack-scheduler/kubernetes/deployment.yaml new file mode 100644 index 00000000..5509572a --- /dev/null +++ b/osdc/modules/bin-pack-scheduler/kubernetes/deployment.yaml @@ -0,0 +1,73 @@ +# bin-pack-scheduler: a stock kube-scheduler run as a secondary scheduler. +# Image tracks eks_version (clusters.yaml); a guard test enforces the minor +# matches. ±1 minor skew is tolerated, so bump on EKS upgrade. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bin-pack-scheduler + namespace: kube-system + labels: + osdc.io/module: bin-pack-scheduler + app: bin-pack-scheduler +spec: + replicas: 2 # HA via leader election; only one schedules at a time + selector: + matchLabels: + app: bin-pack-scheduler + template: + metadata: + labels: + app: bin-pack-scheduler + osdc.io/module: bin-pack-scheduler + spec: + serviceAccountName: bin-pack-scheduler + priorityClassName: system-cluster-critical + # Run on stable infra nodes, not ephemeral runner/workflow nodes. + nodeSelector: + role: base-infrastructure + tolerations: + - key: CriticalAddonsOnly + operator: Equal + value: "true" + effect: NoSchedule + # Spread the 2 replicas across nodes so a single node loss keeps a leader. + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app: bin-pack-scheduler + containers: + - name: kube-scheduler + image: registry.k8s.io/kube-scheduler:v1.35.0 + command: + - kube-scheduler + - --config=/etc/kubernetes/bin-pack-scheduler/config.yaml + - --v=2 + ports: + - name: https + containerPort: 10259 + protocol: TCP + resources: + requests: {cpu: 200m, memory: 256Mi} + limits: {cpu: 500m, memory: 512Mi} + livenessProbe: + httpGet: {path: /healthz, port: 10259, scheme: HTTPS} + initialDelaySeconds: 15 + readinessProbe: + httpGet: {path: /readyz, port: 10259, scheme: HTTPS} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + volumeMounts: + - name: config + mountPath: /etc/kubernetes/bin-pack-scheduler + readOnly: true + volumes: + - name: config + configMap: + name: bin-pack-scheduler-config # kustomize rewrites to the hashed name diff --git a/osdc/modules/bin-pack-scheduler/kubernetes/kustomization.yaml b/osdc/modules/bin-pack-scheduler/kubernetes/kustomization.yaml new file mode 100644 index 00000000..769bb25f --- /dev/null +++ b/osdc/modules/bin-pack-scheduler/kubernetes/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - rbac.yaml + - deployment.yaml + +# Generate the config ConfigMap with a content-hash suffix so the Deployment +# auto-rolls whenever config.yaml changes (no manual rollout restart needed). +configMapGenerator: + - name: bin-pack-scheduler-config + namespace: kube-system + files: + - config.yaml + +generatorOptions: + labels: + osdc.io/module: bin-pack-scheduler diff --git a/osdc/modules/bin-pack-scheduler/kubernetes/rbac.yaml b/osdc/modules/bin-pack-scheduler/kubernetes/rbac.yaml new file mode 100644 index 00000000..11c87acb --- /dev/null +++ b/osdc/modules/bin-pack-scheduler/kubernetes/rbac.yaml @@ -0,0 +1,91 @@ +# RBAC for the bin-pack-scheduler. Reuses the built-in system:kube-scheduler +# and system:volume-scheduler ClusterRoles (track upstream across k8s versions) +# and adds get/update on our own leader-election Lease, which system:kube-scheduler +# scopes to resourceName "kube-scheduler" only. +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: bin-pack-scheduler + namespace: kube-system + labels: + osdc.io/module: bin-pack-scheduler +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: bin-pack-scheduler + labels: + osdc.io/module: bin-pack-scheduler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-scheduler +subjects: + - kind: ServiceAccount + name: bin-pack-scheduler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: bin-pack-scheduler-volume + labels: + osdc.io/module: bin-pack-scheduler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:volume-scheduler +subjects: + - kind: ServiceAccount + name: bin-pack-scheduler + namespace: kube-system +--- +# Lets the scheduler read the extension-apiserver-authentication ConfigMap so +# its HTTPS serving port (10259) can set up delegated auth cleanly on startup. +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: bin-pack-scheduler-auth-reader + namespace: kube-system + labels: + osdc.io/module: bin-pack-scheduler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: bin-pack-scheduler + namespace: kube-system +--- +# Lease create is already granted cluster-wide by system:kube-scheduler; +# get/update on our custom lease name is not, so grant it here. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: bin-pack-scheduler-lease + namespace: kube-system + labels: + osdc.io/module: bin-pack-scheduler +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + resourceNames: ["bin-pack-scheduler"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: bin-pack-scheduler-lease + namespace: kube-system + labels: + osdc.io/module: bin-pack-scheduler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: bin-pack-scheduler-lease +subjects: + - kind: ServiceAccount + name: bin-pack-scheduler + namespace: kube-system diff --git a/osdc/modules/bin-pack-scheduler/scripts/python/test_image_version.py b/osdc/modules/bin-pack-scheduler/scripts/python/test_image_version.py new file mode 100644 index 00000000..6f4eb247 --- /dev/null +++ b/osdc/modules/bin-pack-scheduler/scripts/python/test_image_version.py @@ -0,0 +1,34 @@ +"""Guard: the bin-pack-scheduler image minor must track clusters.yaml eks_version. + +kube-scheduler tolerates only +/-1 minor skew from the API server (the EKS +control-plane version). This pure-logic test fails loudly if someone bumps +eks_version without bumping the scheduler image (or vice-versa), so the two +can't silently diverge across an EKS upgrade. Runs in `just test` — no cluster. +""" + +from pathlib import Path + +import yaml + +_OSDC_ROOT = Path(__file__).resolve().parents[4] +_DEPLOYMENT = _OSDC_ROOT / "modules" / "bin-pack-scheduler" / "kubernetes" / "deployment.yaml" +_CLUSTERS = _OSDC_ROOT / "clusters.yaml" + + +def _scheduler_image_minor(): + """Return the major.minor of the kube-scheduler container image (e.g. '1.35').""" + doc = yaml.safe_load(_DEPLOYMENT.read_text()) + containers = doc["spec"]["template"]["spec"]["containers"] + image = next(c["image"] for c in containers if c["name"] == "kube-scheduler") + tag = image.rsplit(":", 1)[1].lstrip("v") # "v1.35.0" -> "1.35.0" + return ".".join(tag.split(".")[:2]) # -> "1.35" + + +def test_scheduler_image_minor_matches_eks_version(): + eks_version = yaml.safe_load(_CLUSTERS.read_text())["defaults"]["eks_version"] + image_minor = _scheduler_image_minor() + msg = ( + f"bin-pack-scheduler image minor is {image_minor!r} but clusters.yaml " + f"eks_version is {eks_version!r}; bump them together on EKS upgrade." + ) + assert image_minor == eks_version, msg diff --git a/osdc/modules/bin-pack-scheduler/tests/smoke/conftest.py b/osdc/modules/bin-pack-scheduler/tests/smoke/conftest.py new file mode 100644 index 00000000..8dfd070c --- /dev/null +++ b/osdc/modules/bin-pack-scheduler/tests/smoke/conftest.py @@ -0,0 +1 @@ +from smoke_conftest import * # noqa: F403 diff --git a/osdc/modules/bin-pack-scheduler/tests/smoke/test_bin_pack_scheduler.py b/osdc/modules/bin-pack-scheduler/tests/smoke/test_bin_pack_scheduler.py new file mode 100644 index 00000000..34145109 --- /dev/null +++ b/osdc/modules/bin-pack-scheduler/tests/smoke/test_bin_pack_scheduler.py @@ -0,0 +1,26 @@ +"""Smoke tests for the bin-pack-scheduler secondary scheduler. + +Validates that the scheduler Deployment is ready and has acquired its +leader-election Lease (so it is actually scheduling, not just running). +""" + +from __future__ import annotations + +import pytest +from helpers import assert_deployment_ready, run_kubectl + +pytestmark = [pytest.mark.live] + +NAMESPACE = "kube-system" +NAME = "bin-pack-scheduler" + + +class TestBinPackScheduler: + def test_deployment_ready(self, all_deployments: dict) -> None: + assert_deployment_ready(all_deployments, NAMESPACE, NAME) + + def test_leader_lease_held(self) -> None: + """Leader election Lease exists and has a current holder.""" + lease = run_kubectl(["get", "lease", NAME], namespace=NAMESPACE) + holder = lease.get("spec", {}).get("holderIdentity") + assert holder, f"Lease {NAME} has no holderIdentity — no scheduler is leading" From aaa37016909ebe1035454efdb85738fb631d46c2 Mon Sep 17 00:00:00 2001 From: Jean Schmidt Date: Wed, 24 Jun 2026 16:16:30 -0700 Subject: [PATCH 3/5] Broaden bin-pack-scheduler rollout to staging-uw1 and both LF prod clusters [ghstack-poisoned] --- osdc/clusters.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/osdc/clusters.yaml b/osdc/clusters.yaml index d8ef4a7d..52e4e599 100644 --- a/osdc/clusters.yaml +++ b/osdc/clusters.yaml @@ -173,6 +173,7 @@ clusters: - karpenter - arc - nodepools + - bin-pack-scheduler - arc-runners - keda - buildkit @@ -437,6 +438,7 @@ clusters: - karpenter - arc - nodepools + - bin-pack-scheduler - arc-runners - keda - buildkit @@ -480,6 +482,7 @@ clusters: - karpenter - arc - nodepools + - bin-pack-scheduler - arc-runners - keda - buildkit From 52296c543c9b5a9e6154e2fbea110fdad2db84a2 Mon Sep 17 00:00:00 2001 From: Jean Schmidt Date: Wed, 24 Jun 2026 16:22:36 -0700 Subject: [PATCH 4/5] Broaden bin-pack-scheduler rollout to staging-uw1 and both LF prod clusters [ghstack-poisoned] --- osdc/clusters.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/osdc/clusters.yaml b/osdc/clusters.yaml index 52e4e599..bd5af869 100644 --- a/osdc/clusters.yaml +++ b/osdc/clusters.yaml @@ -160,6 +160,7 @@ clusters: github_secret_name: meta-staging-aws-uw1 runner_name_prefix: "c-mt-" # staging/testing prefix — keeps labels distinct from prod's "mt-" runner_group: "meta-staging-aws-uw1" + scheduler_name: bin-pack-scheduler buildkit: arm64_instance_type: m7gd.16xlarge arm64_pods_per_node: 4 @@ -232,6 +233,7 @@ clusters: github_secret_name: meta-staging-aws-ue1 runner_name_prefix: "c-mt-" # staging/testing prefix — keeps labels distinct from prod's "mt-" runner_group: "meta-staging-aws-ue1" + scheduler_name: bin-pack-scheduler buildkit: arm64_instance_type: m7gd.16xlarge arm64_pods_per_node: 4 @@ -421,6 +423,7 @@ clusters: github_secret_name: lf-prod-aws-ue1 runner_name_prefix: "lf-" runner_group: lf-prod-aws-ue1 + scheduler_name: bin-pack-scheduler buildkit: amd64_instance_type: m6id.24xlarge amd64_pods_per_node: 2 @@ -465,6 +468,7 @@ clusters: github_secret_name: lf-prod-aws-ue2 runner_name_prefix: "lf-" runner_group: lf-prod-aws-ue2 + scheduler_name: bin-pack-scheduler buildkit: amd64_instance_type: m6id.24xlarge amd64_pods_per_node: 2 From a0193d4be24fe6c1cb7e928a7aac66c6cc2db1ed Mon Sep 17 00:00:00 2001 From: Jean Schmidt Date: Wed, 24 Jun 2026 17:32:55 -0700 Subject: [PATCH 5/5] Update (base update) [ghstack-poisoned] --- osdc/modules/bin-pack-scheduler/kubernetes/deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/osdc/modules/bin-pack-scheduler/kubernetes/deployment.yaml b/osdc/modules/bin-pack-scheduler/kubernetes/deployment.yaml index 5509572a..1ef70c6e 100644 --- a/osdc/modules/bin-pack-scheduler/kubernetes/deployment.yaml +++ b/osdc/modules/bin-pack-scheduler/kubernetes/deployment.yaml @@ -62,7 +62,9 @@ spec: securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true + runAsGroup: 65532 runAsNonRoot: true + runAsUser: 65532 volumeMounts: - name: config mountPath: /etc/kubernetes/bin-pack-scheduler