diff --git a/api/worker/v1alpha1/generated.pb.go b/api/worker/v1alpha1/generated.pb.go index cb4555878..a12d062fc 100644 --- a/api/worker/v1alpha1/generated.pb.go +++ b/api/worker/v1alpha1/generated.pb.go @@ -6496,11 +6496,6 @@ func (m *ModelDeploymentRoleElasticEP) MarshalToSizedBuffer(dAtA []byte) (int, e _ = i var l int _ = l - i -= len(m.HeadInstanceType) - copy(dAtA[i:], m.HeadInstanceType) - i = encodeVarintGenerated(dAtA, i, uint64(len(m.HeadInstanceType))) - i-- - dAtA[i] = 0x12 i = encodeVarintGenerated(dAtA, i, uint64(m.Width)) i-- dAtA[i] = 0x8 @@ -11446,8 +11441,6 @@ func (m *ModelDeploymentRoleElasticEP) Size() (n int) { var l int _ = l n += 1 + sovGenerated(uint64(m.Width)) - l = len(m.HeadInstanceType) - n += 1 + l + sovGenerated(uint64(l)) return n } @@ -14146,7 +14139,6 @@ func (this *ModelDeploymentRoleElasticEP) String() string { } s := strings.Join([]string{`&ModelDeploymentRoleElasticEP{`, `Width:` + fmt.Sprintf("%v", this.Width) + `,`, - `HeadInstanceType:` + fmt.Sprintf("%v", this.HeadInstanceType) + `,`, `}`, }, "") return s @@ -34671,38 +34663,6 @@ func (m *ModelDeploymentRoleElasticEP) Unmarshal(dAtA []byte) error { break } } - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field HeadInstanceType", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowGenerated - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthGenerated - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthGenerated - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.HeadInstanceType = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/api/worker/v1alpha1/generated.proto b/api/worker/v1alpha1/generated.proto index 928438d5c..5773d1d8f 100644 --- a/api/worker/v1alpha1/generated.proto +++ b/api/worker/v1alpha1/generated.proto @@ -3885,15 +3885,15 @@ message ModelDeploymentRole { // one dedicated logical Ray cluster for the deployment, with a CPU-only control-plane head // and one whole-GPU TP group per DP engine rank. PRESENCE IS THE PROFILE DISCRIMINANT: a role // without it renders exactly as it did before this field existed, and a profile that is - // turned on and off again describes a different deployment, so the field's presence and - // HeadInstanceType are frozen at creation. Width is editable for scale-up; scale-down is + // turned on and off again describes a different deployment. The field's presence is frozen + // at creation. Width is editable for scale-up; scale-down is // refused by admission in this release. // // The profile admits exactly one such role per deployment, running the vLLM engine as a // single Server role of one instance of one Pod. Every member -- the reserved API/DP-master // and every Ray-only worker -- takes a complete TP group declared through ExtraArgs. // All GPUs are whole, unsliced and unpartitioned. Admission binds the resource count to TP. - // The head uses HeadInstanceType's own queue and never counts toward Width. + // The auxiliary head requests no container resources and never counts toward Width. // // +optional optional ModelDeploymentRoleElasticEP elasticEp = 18; @@ -3904,7 +3904,8 @@ message ModelDeploymentRole { // Width is the only mutable field: it answers how large the collective currently should be, // which is a running-state question. This release accepts increases and unchanged values; // admission refuses a decrease until Elastic EP scale-down support is complete. -// Profile presence, HeadInstanceType and the TP degree in ExtraArgs are fixed at creation. +// The role cannot enable or disable this profile after creation. +// Its tensor parallel size comes from ExtraArgs and cannot change after creation. message ModelDeploymentRoleElasticEP { // Width is the total number of GPU engines in the elastic collective, INCLUDING the // reserved API/DP-master member; width-1 of them are Ray-only workers. It is the total @@ -3915,16 +3916,6 @@ message ModelDeploymentRoleElasticEP { // +k8s:validation:minimum=2 // +k8s:validation:maximum=64 optional int32 width = 1; - - // HeadInstanceType is the name of the CPU-only InstanceType the Ray control-plane head - // runs against. The head is admitted as its own CPU-accounted workload on that type's - // queue: it takes no accelerator, joins no GPU queue at zero charge, and is not a width - // member. - // - // +required - // +k8s:validation:minLength=1 - // +k8s:validation:maxLength=253 - optional string headInstanceType = 2; } // ModelDeploymentRoleEndpointsStatus is one role's endpoint eligibility and actual serving diff --git a/api/worker/v1alpha1/model_deployment.go b/api/worker/v1alpha1/model_deployment.go index 5b37bf6ba..36fcea75b 100644 --- a/api/worker/v1alpha1/model_deployment.go +++ b/api/worker/v1alpha1/model_deployment.go @@ -557,15 +557,15 @@ type ModelDeploymentRole struct { // one dedicated logical Ray cluster for the deployment, with a CPU-only control-plane head // and one whole-GPU TP group per DP engine rank. PRESENCE IS THE PROFILE DISCRIMINANT: a role // without it renders exactly as it did before this field existed, and a profile that is - // turned on and off again describes a different deployment, so the field's presence and - // HeadInstanceType are frozen at creation. Width is editable for scale-up; scale-down is + // turned on and off again describes a different deployment. The field's presence is frozen + // at creation. Width is editable for scale-up; scale-down is // refused by admission in this release. // // The profile admits exactly one such role per deployment, running the vLLM engine as a // single Server role of one instance of one Pod. Every member -- the reserved API/DP-master // and every Ray-only worker -- takes a complete TP group declared through ExtraArgs. // All GPUs are whole, unsliced and unpartitioned. Admission binds the resource count to TP. - // The head uses HeadInstanceType's own queue and never counts toward Width. + // The auxiliary head requests no container resources and never counts toward Width. // // +optional ElasticEP *ModelDeploymentRoleElasticEP `json:"elasticEp,omitempty" protobuf:"bytes,18,opt,name=elasticEp"` @@ -576,7 +576,8 @@ type ModelDeploymentRole struct { // Width is the only mutable field: it answers how large the collective currently should be, // which is a running-state question. This release accepts increases and unchanged values; // admission refuses a decrease until Elastic EP scale-down support is complete. -// Profile presence, HeadInstanceType and the TP degree in ExtraArgs are fixed at creation. +// The role cannot enable or disable this profile after creation. +// Its tensor parallel size comes from ExtraArgs and cannot change after creation. type ModelDeploymentRoleElasticEP struct { // Width is the total number of GPU engines in the elastic collective, INCLUDING the // reserved API/DP-master member; width-1 of them are Ray-only workers. It is the total @@ -587,16 +588,6 @@ type ModelDeploymentRoleElasticEP struct { // +k8s:validation:minimum=2 // +k8s:validation:maximum=64 Width int32 `json:"width" protobuf:"varint,1,name=width"` - - // HeadInstanceType is the name of the CPU-only InstanceType the Ray control-plane head - // runs against. The head is admitted as its own CPU-accounted workload on that type's - // queue: it takes no accelerator, joins no GPU queue at zero charge, and is not a width - // member. - // - // +required - // +k8s:validation:minLength=1 - // +k8s:validation:maxLength=253 - HeadInstanceType string `json:"headInstanceType" protobuf:"bytes,2,name=headInstanceType"` } // ModelDeploymentRoleTopology is the topology request for one independent replica group. diff --git a/api/worker/v1alpha1/zz_generated.crds.go b/api/worker/v1alpha1/zz_generated.crds.go index 86bcd6413..725984a54 100644 --- a/api/worker/v1alpha1/zz_generated.crds.go +++ b/api/worker/v1alpha1/zz_generated.crds.go @@ -4824,19 +4824,12 @@ func crd_gpustack_api_worker_v1alpha1_ModelDeployment() *v1.CustomResourceDefini XListType: ptr.To[string]("atomic"), }, "elasticEp": { - Description: "ElasticEP opts this role into the managed elastic-EP profile: the operator then renders\none dedicated logical Ray cluster for the deployment, with a CPU-only control-plane head\nand one whole-GPU TP group per DP engine rank. PRESENCE IS THE PROFILE DISCRIMINANT: a role\nwithout it renders exactly as it did before this field existed, and a profile that is\nturned on and off again describes a different deployment, so the field's presence and\nHeadInstanceType are frozen at creation. Width is editable for scale-up; scale-down is\nrefused by admission in this release.\nThe profile admits exactly one such role per deployment, running the vLLM engine as a\nsingle Server role of one instance of one Pod. Every member -- the reserved API/DP-master\nand every Ray-only worker -- takes a complete TP group declared through ExtraArgs.\nAll GPUs are whole, unsliced and unpartitioned. Admission binds the resource count to TP.\nThe head uses HeadInstanceType's own queue and never counts toward Width.", + Description: "ElasticEP opts this role into the managed elastic-EP profile: the operator then renders\none dedicated logical Ray cluster for the deployment, with a CPU-only control-plane head\nand one whole-GPU TP group per DP engine rank. PRESENCE IS THE PROFILE DISCRIMINANT: a role\nwithout it renders exactly as it did before this field existed, and a profile that is\nturned on and off again describes a different deployment. The field's presence is frozen\nat creation. Width is editable for scale-up; scale-down is\nrefused by admission in this release.\nThe profile admits exactly one such role per deployment, running the vLLM engine as a\nsingle Server role of one instance of one Pod. Every member -- the reserved API/DP-master\nand every Ray-only worker -- takes a complete TP group declared through ExtraArgs.\nAll GPUs are whole, unsliced and unpartitioned. Admission binds the resource count to TP.\nThe auxiliary head requests no container resources and never counts toward Width.", Type: "object", Required: []string{ "width", - "headInstanceType", }, Properties: map[string]v1.JSONSchemaProps{ - "headInstanceType": { - Description: "HeadInstanceType is the name of the CPU-only InstanceType the Ray control-plane head\nruns against. The head is admitted as its own CPU-accounted workload on that type's\nqueue: it takes no accelerator, joins no GPU queue at zero charge, and is not a width\nmember.", - Type: "string", - MaxLength: ptr.To[int64](253), - MinLength: ptr.To[int64](1), - }, "width": { Description: "Width is the total number of GPU engines in the elastic collective, INCLUDING the\nreserved API/DP-master member; width-1 of them are Ray-only workers. It is the total\nengine world the engine is told to run, not a Pod count and not a rank mapping: which\nmember holds which rank is the engine's own runtime fact and is never implied by this\nnumber.", Type: "integer", diff --git a/api/worker/zz_generated.openapi.go b/api/worker/zz_generated.openapi.go index 99d7d1493..1a59f8bea 100644 --- a/api/worker/zz_generated.openapi.go +++ b/api/worker/zz_generated.openapi.go @@ -10051,7 +10051,7 @@ func schema_gpustack_api_worker_v1alpha1_ModelDeploymentRole(ref common.Referenc }, "elasticEp": { SchemaProps: spec.SchemaProps{ - Description: "ElasticEP opts this role into the managed elastic-EP profile: the operator then renders one dedicated logical Ray cluster for the deployment, with a CPU-only control-plane head and one whole-GPU TP group per DP engine rank. PRESENCE IS THE PROFILE DISCRIMINANT: a role without it renders exactly as it did before this field existed, and a profile that is turned on and off again describes a different deployment, so the field's presence and HeadInstanceType are frozen at creation. Width is editable for scale-up; scale-down is refused by admission in this release.\n\nThe profile admits exactly one such role per deployment, running the vLLM engine as a single Server role of one instance of one Pod. Every member -- the reserved API/DP-master and every Ray-only worker -- takes a complete TP group declared through ExtraArgs. All GPUs are whole, unsliced and unpartitioned. Admission binds the resource count to TP. The head uses HeadInstanceType's own queue and never counts toward Width.", + Description: "ElasticEP opts this role into the managed elastic-EP profile: the operator then renders one dedicated logical Ray cluster for the deployment, with a CPU-only control-plane head and one whole-GPU TP group per DP engine rank. PRESENCE IS THE PROFILE DISCRIMINANT: a role without it renders exactly as it did before this field existed, and a profile that is turned on and off again describes a different deployment. The field's presence is frozen at creation. Width is editable for scale-up; scale-down is refused by admission in this release.\n\nThe profile admits exactly one such role per deployment, running the vLLM engine as a single Server role of one instance of one Pod. Every member -- the reserved API/DP-master and every Ray-only worker -- takes a complete TP group declared through ExtraArgs. All GPUs are whole, unsliced and unpartitioned. Admission binds the resource count to TP. The auxiliary head requests no container resources and never counts toward Width.", Ref: ref(v1alpha1.ModelDeploymentRoleElasticEP{}.OpenAPIModelName()), }, }, @@ -10068,7 +10068,7 @@ func schema_gpustack_api_worker_v1alpha1_ModelDeploymentRoleElasticEP(ref common return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "ModelDeploymentRoleElasticEP is the elastic-EP profile of one role.\n\nWidth is the only mutable field: it answers how large the collective currently should be, which is a running-state question. This release accepts increases and unchanged values; admission refuses a decrease until Elastic EP scale-down support is complete. Profile presence, HeadInstanceType and the TP degree in ExtraArgs are fixed at creation.", + Description: "ModelDeploymentRoleElasticEP is the elastic-EP profile of one role.\n\nWidth is the only mutable field: it answers how large the collective currently should be, which is a running-state question. This release accepts increases and unchanged values; admission refuses a decrease until Elastic EP scale-down support is complete. The role cannot enable or disable this profile after creation. Its tensor parallel size comes from ExtraArgs and cannot change after creation.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "width": { @@ -10081,18 +10081,8 @@ func schema_gpustack_api_worker_v1alpha1_ModelDeploymentRoleElasticEP(ref common Format: "int32", }, }, - "headInstanceType": { - SchemaProps: spec.SchemaProps{ - Description: "HeadInstanceType is the name of the CPU-only InstanceType the Ray control-plane head runs against. The head is admitted as its own CPU-accounted workload on that type's queue: it takes no accelerator, joins no GPU queue at zero charge, and is not a width member.", - Default: "", - MinLength: ptr.To[int64](1), - MaxLength: ptr.To[int64](253), - Type: []string{"string"}, - Format: "", - }, - }, }, - Required: []string{"width", "headInstanceType"}, + Required: []string{"width"}, }, }, } diff --git a/docs/modules/model-deployment/deployment.md b/docs/modules/model-deployment/deployment.md index 53a603834..828c71c8d 100644 --- a/docs/modules/model-deployment/deployment.md +++ b/docs/modules/model-deployment/deployment.md @@ -298,11 +298,20 @@ webhook-level one, so nothing here should be read as a claim about how that issu ### Shared memory -`roles[].shmSize` sets the capacity limit of `/dev/shm` in each role Pod. It accepts a positive Kubernetes +`spec.roles[].shmSize` sets the capacity limit of `/dev/shm` in each role Pod. It accepts a positive Kubernetes quantity, such as `32Gi`. Omission renders a `16Gi` memory-backed `EmptyDir`. This applies to both engines, take-over roles, and the Elastic Ray head and GPU members. Each Pod has its own volume. Only the main container mounts it. +Set it in the role configuration: + +```yaml +spec: + roles: + - name: server + shmSize: 32Gi +``` + An explicit `/dev/shm` mount in `additionalVolumes` takes precedence, including when `shmSize` is set. The operator preserves that mount's capacity and backing. The default is a project capacity choice, not a universal engine requirement. diff --git a/docs/modules/model-deployment/elastic-ep.md b/docs/modules/model-deployment/elastic-ep.md index 11a796b58..be454326c 100644 --- a/docs/modules/model-deployment/elastic-ep.md +++ b/docs/modules/model-deployment/elastic-ep.md @@ -19,12 +19,11 @@ The engine reconfigures while Kubernetes, Kueue, Ray and accelerator accounting ## What the profile means -`roles[].elasticEp` selects this mode for a role. Two fields matter: +`roles[].elasticEp` selects this mode for a role. Its only field sets the target width: | Field | Meaning | |---|---| | `width` | Total GPU engines in the collective, counting the reserved master. Mutable, from 2 to 64. | -| `headInstanceType` | The CPU-only InstanceType the Ray control plane runs on. Set once. | `width` counts DP engines, including the master. Each member has its own GPU Pod. Set TP through the role's `extraArgs`, using the engine's `--tensor-parallel-size` argument. @@ -47,7 +46,7 @@ enforces the rest: - there is no initial fabric to declare. Adding or removing the profile on an existing role is refused. -`headInstanceType` and the effective TP value are fixed after creation. +The effective TP value is fixed after creation. Equivalent TP argument spellings are accepted. Omitting TP and explicitly setting TP=1 are equivalent. A width change adjusts DP and EP while preserving TP. Online TP changes are refused. @@ -56,15 +55,26 @@ is why the master is counted inside `width` and is never retired. ## Requirements -The engine image must be a vLLM 0.29.0 image that already contains a full Ray installation. The +The recorded runtime baseline uses a vLLM 0.29.0 image with a full Ray installation. The official vLLM image of that version ships without Ray, and the profile will not start on it. The pinned requirement in the resize spec is Ray 2.56.1 with the full default closure. The TP size must fit the selected MoE model, engine kernels, and GPUs available on one node. The operator does not limit TP to the size used by a particular test. +Static TP=2 inference has passed; online TP>1 resizing remains under investigation. +Admission of TP=2, 4 or 8 does not establish dynamic inference support. -The Ray control plane needs its own Pod, which is why `headInstanceType` is required. It names an -InstanceType that must already exist and must be CPU-only. +The operator directly manages a separate CPU Ray head Pod and its Service. +The head has no InstanceType or LocalQueue dependency and does not count toward GPU width. +Its Pod template declares no container or Pod resource requests or limits. +Kubernetes schedules it using the namespace's normal admission rules. +A LimitRange or another webhook can still inject resources at admission. + +GPU master and worker Pods retain their GPU requests and device allocations. +Their `ray start` commands let Ray discover container-visible GPUs. +The CPU head uses `--num-gpus=0` to exclude it from GPU tasks. +The existing readiness check requires each live GPU Ray node's capacity to equal TP. +Ray's logical count does not replace the allocated-device and Pod identity checks. The profile increases the number of GPU members. It does not resize the underlying nodes or claim cloud node elasticity, and it does not patch anything upstream in the engine. It drives the engine @@ -103,7 +113,6 @@ spec: image: your-registry/vllm-ray:0.29.0 elasticEp: width: 2 - headInstanceType: elastic-cpu ports: - name: http port: 8000 @@ -117,6 +126,21 @@ To use TP=2, add `extraArgs: ["--tensor-parallel-size", "2"]` and request `resou If the accelerator quantity is omitted, admission defaults it to the TP size. Explicit quantities must equal the TP size. Each member's CPU and memory scale with its GPU count. +### Maximum width + +Starting with vLLM 0.30.0, set the largest planned width through `extraArgs` when creating the deployment: + +```yaml +extraArgs: + - --elastic-ep-max-dp-size + - "4" +``` + +This example permits widths up to 4. The maximum is fixed after creation. +Create a new deployment to change it. +In vLLM 0.30.0 and later, omitting this argument prevents expansion beyond the initial width. +Older vLLM releases do not accept this argument. + ## Changing the width Change only `width`. Starting at 2, this patch requests 4. Wait for the resize to complete before @@ -256,10 +280,10 @@ The operator retains existing elastic Pods, so an env edit does not change their ## Limits -- The engine must be a vLLM 0.29.0 image with a full Ray installation supplied by you. +- This managed Elastic profile supports vLLM with a full Ray installation. + SGLang supports ordinary replica scaling; its native elastic protocol is not integrated. - Width is bounded to 2 through 64. -- The profile cannot be added to or removed from an existing role, and `headInstanceType` cannot - change. +- The profile cannot be added to or removed from an existing role. - There is one serving instance per deployment. `replicas` and `size` are not a width mechanism. - Only the master serves HTTP. - No RDMA fabric is requested for this profile. diff --git a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modeldeploymentrole.go b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modeldeploymentrole.go index 70c2edc71..48bd75510 100644 --- a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modeldeploymentrole.go +++ b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modeldeploymentrole.go @@ -194,15 +194,15 @@ type ModelDeploymentRoleApplyConfiguration struct { // one dedicated logical Ray cluster for the deployment, with a CPU-only control-plane head // and one whole-GPU TP group per DP engine rank. PRESENCE IS THE PROFILE DISCRIMINANT: a role // without it renders exactly as it did before this field existed, and a profile that is - // turned on and off again describes a different deployment, so the field's presence and - // HeadInstanceType are frozen at creation. Width is editable for scale-up; scale-down is + // turned on and off again describes a different deployment. The field's presence is frozen + // at creation. Width is editable for scale-up; scale-down is // refused by admission in this release. // // The profile admits exactly one such role per deployment, running the vLLM engine as a // single Server role of one instance of one Pod. Every member -- the reserved API/DP-master // and every Ray-only worker -- takes a complete TP group declared through ExtraArgs. // All GPUs are whole, unsliced and unpartitioned. Admission binds the resource count to TP. - // The head uses HeadInstanceType's own queue and never counts toward Width. + // The auxiliary head requests no container resources and never counts toward Width. ElasticEP *ModelDeploymentRoleElasticEPApplyConfiguration `json:"elasticEp,omitempty"` } diff --git a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modeldeploymentroleelasticep.go b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modeldeploymentroleelasticep.go index 6aab5ff82..498e44fed 100644 --- a/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modeldeploymentroleelasticep.go +++ b/pkg/kubeclients/applyconfiguration/worker/v1alpha1/modeldeploymentroleelasticep.go @@ -10,7 +10,8 @@ package v1alpha1 // Width is the only mutable field: it answers how large the collective currently should be, // which is a running-state question. This release accepts increases and unchanged values; // admission refuses a decrease until Elastic EP scale-down support is complete. -// Profile presence, HeadInstanceType and the TP degree in ExtraArgs are fixed at creation. +// The role cannot enable or disable this profile after creation. +// Its tensor parallel size comes from ExtraArgs and cannot change after creation. type ModelDeploymentRoleElasticEPApplyConfiguration struct { // Width is the total number of GPU engines in the elastic collective, INCLUDING the // reserved API/DP-master member; width-1 of them are Ray-only workers. It is the total @@ -18,11 +19,6 @@ type ModelDeploymentRoleElasticEPApplyConfiguration struct { // member holds which rank is the engine's own runtime fact and is never implied by this // number. Width *int32 `json:"width,omitempty"` - // HeadInstanceType is the name of the CPU-only InstanceType the Ray control-plane head - // runs against. The head is admitted as its own CPU-accounted workload on that type's - // queue: it takes no accelerator, joins no GPU queue at zero charge, and is not a width - // member. - HeadInstanceType *string `json:"headInstanceType,omitempty"` } // ModelDeploymentRoleElasticEPApplyConfiguration constructs a declarative configuration of the ModelDeploymentRoleElasticEP type for use with @@ -38,11 +34,3 @@ func (b *ModelDeploymentRoleElasticEPApplyConfiguration) WithWidth(value int32) b.Width = &value return b } - -// WithHeadInstanceType sets the HeadInstanceType field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the HeadInstanceType field is set to the value of the last call. -func (b *ModelDeploymentRoleElasticEPApplyConfiguration) WithHeadInstanceType(value string) *ModelDeploymentRoleElasticEPApplyConfiguration { - b.HeadInstanceType = &value - return b -} diff --git a/pkg/worker/controllers/worker/model_deployment.go b/pkg/worker/controllers/worker/model_deployment.go index de0199034..e55ca03e8 100644 --- a/pkg/worker/controllers/worker/model_deployment.go +++ b/pkg/worker/controllers/worker/model_deployment.go @@ -1872,7 +1872,12 @@ func (r *ModelDeploymentReconciler) modelDeploymentTakenGroups( if !includeTerminal && (pod.Status.Phase == core.PodSucceeded || pod.Status.Phase == core.PodFailed) { continue } - if group := pod.Labels[kueuepodconst.GroupNameLabel]; group != "" { + group := pod.Labels[kueuepodconst.GroupNameLabel] + if group == "" && ModelDeploymentElasticRole(md) != nil && role == modelDeploymentElasticHeadName(md) { + // The auxiliary head has no Kueue group, but its uncached presence still holds the slot. + group = modelDeploymentReplicaGroupName(md, role, 0) + } + if group != "" { taken.Insert(group) } } diff --git a/pkg/worker/controllers/worker/model_deployment_elastic_allocation_test.go b/pkg/worker/controllers/worker/model_deployment_elastic_allocation_test.go index 3944599ba..cf5b3766d 100644 --- a/pkg/worker/controllers/worker/model_deployment_elastic_allocation_test.go +++ b/pkg/worker/controllers/worker/model_deployment_elastic_allocation_test.go @@ -39,7 +39,7 @@ func allocationDeployment() *workercore.ModelDeployment { md := newRenderDeployment(func(md *workercore.ModelDeployment) { md.Generation = allocationGeneration md.Spec.KVCache = nil - md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 4, HeadInstanceType: "cpu"} + md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 4} }) md.UID = allocationDeployUID diff --git a/pkg/worker/controllers/worker/model_deployment_elastic_realization.go b/pkg/worker/controllers/worker/model_deployment_elastic_realization.go index 83a9cb4b5..7c03a94bb 100644 --- a/pkg/worker/controllers/worker/model_deployment_elastic_realization.go +++ b/pkg/worker/controllers/worker/model_deployment_elastic_realization.go @@ -11,6 +11,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" ctrllog "sigs.k8s.io/controller-runtime/pkg/log" workercore "gpustack.ai/gpustack/api/worker/v1alpha1" @@ -50,7 +51,7 @@ func (r *ModelDeploymentReconciler) renderModelDeploymentElasticPods(ctx context connection *ModelDeploymentConnectorInput, protocols []string, weights *modelArtifactWeights, ) (map[string]map[int][]*core.Pod, error) { role := ModelDeploymentElasticRole(md) - tpSize, err := ModelDeploymentElasticTensorParallelSize(role) + _, err := ModelDeploymentElasticTensorParallelSize(role) if err != nil { return nil, err } @@ -107,9 +108,8 @@ func (r *ModelDeploymentReconciler) renderModelDeploymentElasticPods(ctx context }}, core.EnvVar{Name: "VLLM_RAY_DP_PACK_STRATEGY", Value: "strict"}, ) - tp := strconv.Itoa(tpSize) join := "ray start --node-ip-address=\"$GPUSTACK_ELASTIC_POD_IP\" --address=" + address + - " --num-gpus=" + tp + " --labels=" + ModelDeploymentElasticRayNodeIdentityLabel + "=\"$GPUSTACK_ELASTIC_POD_UID\"" + " --labels=" + ModelDeploymentElasticRayNodeIdentityLabel + "=\"$GPUSTACK_ELASTIC_POD_UID\"" if ordinal == 0 { argv := slices.Clone(c.Command) @@ -133,17 +133,21 @@ func (r *ModelDeploymentReconciler) renderModelDeploymentElasticPods(ctx context } pod.Annotations[modelDeploymentPodSpecHashAnnotation] = modelDeploymentPodSpecHash(pod) } - head := gpu.DeepCopy() - head.Name = modelDeploymentElasticHeadName(md) - head.InstanceType = role.ElasticEP.HeadInstanceType - head.Resources = nil - head.Replicas = 1 - head.ExtraArgs = nil - head.Ports = nil - head.Command = []string{"/bin/sh", "-ec", ModelDeploymentElasticHeadCommand + - " --num-gpus=0 --port=6379 --dashboard-host=0.0.0.0 --labels=" + - ModelDeploymentElasticRayNodeIdentityLabel + "=\"$GPUSTACK_ELASTIC_POD_UID\" --block"} - head.Env = slices.DeleteFunc(head.Env, func(env workercore.ModelDeploymentEnvVar) bool { + headPod := renderModelDeploymentElasticHeadPod(md, gpu, desired[role.Name][0][0].Spec.Containers[0]) + desired[modelDeploymentElasticHeadName(md)] = map[int][]*core.Pod{0: {headPod}} + return desired, nil +} + +// renderModelDeploymentElasticHeadPod bypasses GPU and InstanceType admission for the auxiliary head. +func renderModelDeploymentElasticHeadPod( + md *workercore.ModelDeployment, role *workercore.ModelDeploymentRole, main core.Container, +) *core.Pod { + name := modelDeploymentElasticHeadName(md) + labels := modelDeploymentSelectorLabels(md, &workercore.ModelDeploymentRole{Name: name}) + labels["app.kubernetes.io/part-of"] = "gpustack-operator-worker" + labels[modelDeploymentReplicaOrdinalLabel] = "0" + labels[modelDeploymentMemberIndexLabel] = "0" + env := slices.DeleteFunc(slices.Clone(main.Env), func(env core.EnvVar) bool { switch env.Name { case "RAY_LOG_TO_STDERR", "RAY_LOGGER_LEVEL", "RAY_BACKEND_LOG_LEVEL", "RAY_DEDUP_LOGS": return false @@ -151,21 +155,37 @@ func (r *ModelDeploymentReconciler) renderModelDeploymentElasticPods(ctx context return true } }) - headMD := md.DeepCopy() - headMD.Spec.Roles = []workercore.ModelDeploymentRole{*head} - heads, err := r.renderModelDeploymentPods(ctx, headMD, nil, nil, nil) - if err != nil { - return nil, err - } - headPod := heads[head.Name][0][0] - headPod.Spec.Containers[0].Env = append(headPod.Spec.Containers[0].Env, core.EnvVar{ - Name: "GPUSTACK_ELASTIC_POD_UID", ValueFrom: &core.EnvVarSource{ - FieldRef: &core.ObjectFieldSelector{FieldPath: "metadata.uid"}, + env = append(env, core.EnvVar{Name: "GPUSTACK_ELASTIC_POD_UID", ValueFrom: &core.EnvVarSource{ + FieldRef: &core.ObjectFieldSelector{FieldPath: "metadata.uid"}, + }}) + vols, mounts := convertModelDeploymentAdditionalVolumes(role.AdditionalVolumes) + vols, mounts = modelDeploymentSharedMemoryVolumes(role, vols, mounts) + pod := &core.Pod{ + ObjectMeta: meta.ObjectMeta{GenerateName: md.Name + "-" + name + "-", Namespace: md.Namespace, Labels: labels}, + Spec: core.PodSpec{ + AutomountServiceAccountToken: ptr.To(false), + EnableServiceLinks: ptr.To(false), + RestartPolicy: core.RestartPolicyAlways, + ImagePullSecrets: role.ImagePullSecrets, + Volumes: vols, + Containers: []core.Container{{ + Name: modelDeploymentMainContainerName, Image: main.Image, ImagePullPolicy: role.ImagePullPolicy, + SecurityContext: main.SecurityContext.DeepCopy(), + Command: []string{"/bin/sh", "-ec", ModelDeploymentElasticHeadCommand + + " --num-gpus=0 --port=6379 --dashboard-host=0.0.0.0 --labels=" + + ModelDeploymentElasticRayNodeIdentityLabel + "=\"$GPUSTACK_ELASTIC_POD_UID\" --block"}, + Env: env, VolumeMounts: mounts, + Ports: []core.ContainerPort{{Name: "gcs", ContainerPort: 6379}, {Name: "dashboard", ContainerPort: 8265}}, + }}, }, - }) - headPod.Annotations[modelDeploymentPodSpecHashAnnotation] = modelDeploymentPodSpecHash(headPod) - desired[head.Name] = heads[head.Name] - return desired, nil + } + if role.TerminationGracePeriodSeconds != nil { + pod.Spec.TerminationGracePeriodSeconds = ptr.To(*role.TerminationGracePeriodSeconds) + } + systemmeta.NoteResource(pod, ModelDeploymentResourceType, map[string]string{ModelDeploymentResourceNoteRole: name}) + kubemeta.ControlOnWithoutBlock(pod, md, workercore.SchemeGroupVersionKind("ModelDeployment")) + pod.Annotations[modelDeploymentPodSpecHashAnnotation] = modelDeploymentPodSpecHash(pod) + return pod } func renderModelDeploymentElasticHeadService(md *workercore.ModelDeployment) *core.Service { diff --git a/pkg/worker/controllers/worker/model_deployment_elastic_reconcile_test.go b/pkg/worker/controllers/worker/model_deployment_elastic_reconcile_test.go index 90d2c4c23..53b84eb15 100644 --- a/pkg/worker/controllers/worker/model_deployment_elastic_reconcile_test.go +++ b/pkg/worker/controllers/worker/model_deployment_elastic_reconcile_test.go @@ -24,7 +24,6 @@ import ( ctrlinterceptor "sigs.k8s.io/controller-runtime/pkg/client/interceptor" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta2" - workerapi "gpustack.ai/gpustack/api/worker/v1" workercore "gpustack.ai/gpustack/api/worker/v1alpha1" "gpustack.ai/gpustack/pkg/deviceplugin" "gpustack.ai/gpustack/pkg/kubeapistatus" @@ -80,16 +79,11 @@ func newElasticConvergenceFixture(t *testing.T) *elasticConvergenceFixture { md.Spec.Engine.Version = "0.29.0" md.Spec.Roles[0].Replicas = 1 md.Spec.Roles[0].Resources = &workercore.ModelDeploymentRoleResources{Accelerator: resource.NewQuantity(1, resource.DecimalSI)} - md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 2, HeadInstanceType: "cpu"} - cpu := newRenderInstanceType(func(it *workerapi.InstanceType) { - it.Name = "cpu" - it.Spec.Acceleratable = false - it.Status.Entrance = "cpu-queue" - }) + md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 2} md.ResourceVersion = "1" cli := ctrlfake.NewClientBuilder().WithScheme(scheme.Scheme). WithObjectTracker(clitesting.NewObjectTracker(scheme.Scheme, scheme.Codecs.UniversalDecoder())). - WithObjects(md, newRenderInstanceType(), cpu). + WithObjects(md, newRenderInstanceType()). WithStatusSubresource(&workercore.ModelDeployment{}, &workercore.Devices{}). WithIndex(&core.Pod{}, "spec.nodeName", func(obj ctrlcli.Object) []string { return []string{obj.(*core.Pod).Spec.NodeName} }). WithIndex(&core.Pod{}, modelDeploymentDrainIndexPodUID, func(obj ctrlcli.Object) []string { return []string{string(obj.GetUID())} }).Build() diff --git a/pkg/worker/controllers/worker/model_deployment_elastic_workload_test.go b/pkg/worker/controllers/worker/model_deployment_elastic_workload_test.go index 73d5f48dd..5519c5ec0 100644 --- a/pkg/worker/controllers/worker/model_deployment_elastic_workload_test.go +++ b/pkg/worker/controllers/worker/model_deployment_elastic_workload_test.go @@ -23,6 +23,7 @@ import ( worker "gpustack.ai/gpustack/api/worker/v1" workercore "gpustack.ai/gpustack/api/worker/v1alpha1" "gpustack.ai/gpustack/pkg/nodefeature" + "gpustack.ai/gpustack/pkg/systemmeta" "gpustack.ai/gpustack/pkg/worker/elasticprofile" ) @@ -60,7 +61,7 @@ func TestElasticRayLoggingEnvironment(t *testing.T) { if !tc.ordinary { md.Spec.Engine.Version = "0.29.0" md.Spec.Roles[0].Replicas = 1 - md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 2, HeadInstanceType: "cpu"} + md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 2} } cpu := newRenderInstanceType(func(it *worker.InstanceType) { it.Name = "cpu" @@ -111,14 +112,9 @@ func TestElasticConvergenceCreatesChildren(t *testing.T) { md.Spec.Engine.Version = "0.29.0" md.Spec.Roles[0].Replicas = 1 md.Spec.Roles[0].Resources = &workercore.ModelDeploymentRoleResources{Accelerator: resource.NewQuantity(int64(tp), resource.DecimalSI)} - md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 2, HeadInstanceType: "cpu"} + md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 2} md.Spec.Roles[0].ExtraArgs = []string{"--tensor-parallel-size", strconv.Itoa(tp)} - cpu := newRenderInstanceType(func(it *worker.InstanceType) { - it.Name = "cpu" - it.Spec.Acceleratable = false - it.Status.Entrance = "cpu-queue" - }) - cli := newModelDeploymentClient(md, newRenderInstanceType(), cpu) + cli := newModelDeploymentClient(md, newRenderInstanceType()) r := &ModelDeploymentReconciler{Client: cli, APIReader: cli} gpuType := new(worker.InstanceType) require.NoError(t, cli.Get(context.Background(), ctrlcli.ObjectKey{Name: md.Spec.Roles[0].InstanceType}, gpuType)) @@ -134,17 +130,25 @@ func TestElasticConvergenceCreatesChildren(t *testing.T) { var master core.Pod for i := range pods.Items { pod := &pods.Items[i] - groups[pod.Labels[kueuepodconst.GroupNameLabel]] = true + if group := pod.Labels[kueuepodconst.GroupNameLabel]; group != "" { + groups[group] = true + } pod.UID = types.UID(fmt.Sprintf("uid-%d", i)) c := pod.Spec.Containers[0] if modelDeploymentPodRole(pod) == modelDeploymentElasticHeadName(md) { - require.Equal(t, "cpu-queue", pod.Labels["kueue.x-k8s.io/queue-name"]) - require.NotContains(t, c.Resources.Limits, gpuResource) + require.NotContains(t, pod.Labels, "kueue.x-k8s.io/queue-name") + require.NotContains(t, pod.Labels, kueuepodconst.GroupNameLabel) + require.Empty(t, pod.Spec.InitContainers) + require.Nil(t, pod.Spec.Resources) + for _, container := range pod.Spec.Containers { + require.Empty(t, container.Resources.Requests) + require.Empty(t, container.Resources.Limits) + } require.Contains(t, c.Command[2], "--num-gpus=0") } else { q := c.Resources.Limits[gpuResource] require.Equal(t, int64(tp), q.Value()) - require.Contains(t, c.Command[2], fmt.Sprintf("--num-gpus=%d", tp)) + require.NotContains(t, c.Command[2], "--num-gpus") require.Equal(t, unitCPU.MilliValue()*int64(tp), c.Resources.Limits.Cpu().MilliValue()) require.Equal(t, unitMemory.Value()*int64(tp), c.Resources.Limits.Memory().Value()) require.Contains(t, c.Command[2], "gpustack-pod-uid=\"$GPUSTACK_ELASTIC_POD_UID\"") @@ -159,7 +163,7 @@ func TestElasticConvergenceCreatesChildren(t *testing.T) { } require.NoError(t, cli.Update(context.Background(), pod)) } - require.Len(t, groups, 3) + require.Len(t, groups, 2, "only GPU members use Kueue groups") require.NotEmpty(t, master.Name) services := new(core.ServiceList) require.NoError(t, cli.List(context.Background(), services, ctrlcli.InNamespace(md.Namespace))) @@ -435,7 +439,7 @@ func TestModelDeploymentRolesSharedMemory(t *testing.T) { md.Spec.KVCache = nil md.Spec.Roles[0].Replicas = 1 if !tc.ordinary { - md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 2, HeadInstanceType: "cpu"} + md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 2} } md.Spec.Roles[0].AdditionalVolumes = tc.additional if tc.engine != "" { @@ -571,7 +575,7 @@ func TestElasticUnknownBootstrapRetainsMembers(t *testing.T) { md := newRenderDeployment() md.Spec.KVCache = nil md.Spec.Roles[0].Replicas = 1 - md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 2, HeadInstanceType: "cpu"} + md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 2} cpu := newRenderInstanceType(func(it *worker.InstanceType) { it.Name = "cpu"; it.Spec.Acceleratable = false }) cli := newModelDeploymentClient(md, newRenderInstanceType(), cpu) r := &ModelDeploymentReconciler{Client: cli, APIReader: cli} @@ -600,13 +604,12 @@ func TestElasticCreateUsesUncachedOccupancy(t *testing.T) { md := newRenderDeployment() md.Spec.KVCache = nil md.Spec.Roles[0].Replicas = 1 - md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 2, HeadInstanceType: "cpu"} - cpu := newRenderInstanceType(func(it *worker.InstanceType) { it.Name = "cpu"; it.Spec.Acceleratable = false }) - fresh := newModelDeploymentClient(md, newRenderInstanceType(), cpu) + md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 2} + fresh := newModelDeploymentClient(md, newRenderInstanceType()) r := &ModelDeploymentReconciler{Client: fresh, APIReader: fresh} _, err := r.convergeModelDeployment(context.Background(), md) require.NoError(t, err) - stale := newModelDeploymentClient(md, newRenderInstanceType(), cpu) + stale := newModelDeploymentClient(md, newRenderInstanceType()) r.Client = stale _, err = r.convergeModelDeployment(context.Background(), md) require.NoError(t, err) @@ -615,6 +618,124 @@ func TestElasticCreateUsesUncachedOccupancy(t *testing.T) { require.Empty(t, pods.Items, "a lost create response must not create a second member") } +func TestElasticAuxiliaryHeadRoleSettings(t *testing.T) { + for _, tc := range []struct { + name string + privileged bool + grace *int64 + }{ + {name: "defaults"}, + {name: "privileged", privileged: true}, + {name: "explicit grace", grace: ptr.To(int64(120))}, + {name: "privileged with explicit grace", privileged: true, grace: ptr.To(int64(120))}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + md := newRenderDeployment() + md.Spec.KVCache = nil + md.Spec.Roles[0].Replicas = 1 + md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{Width: 2} + md.Spec.Roles[0].Privileged = tc.privileged + md.Spec.Roles[0].TerminationGracePeriodSeconds = tc.grace + cli := newModelDeploymentClient(md, newRenderInstanceType()) + r := &ModelDeploymentReconciler{Client: cli, APIReader: cli} + _, err := r.convergeModelDeployment(ctx, md) + require.NoError(t, err) + pods := new(core.PodList) + require.NoError(t, cli.List(ctx, pods)) + heads := 0 + for _, pod := range pods.Items { + if modelDeploymentPodRole(&pod) != modelDeploymentElasticHeadName(md) { + continue + } + heads++ + container := pod.Spec.Containers[0] + if tc.privileged { + require.NotNil(t, container.SecurityContext) + require.NotNil(t, container.SecurityContext.Privileged) + require.True(t, *container.SecurityContext.Privileged) + } else { + require.Nil(t, container.SecurityContext) + } + require.Equal(t, tc.grace, pod.Spec.TerminationGracePeriodSeconds) + require.Nil(t, pod.Spec.RuntimeClassName) + require.Nil(t, pod.Spec.SecurityContext) + require.Empty(t, container.Resources.Requests) + require.Empty(t, container.Resources.Limits) + } + require.Equal(t, 1, heads) + }) + } +} + +func TestElasticAuxiliaryHeadLifecycle(t *testing.T) { + for _, tc := range []struct { + name string + phase core.PodPhase + teardown bool + retained bool + }{ + {name: "retain legacy running head", phase: core.PodRunning, retained: true}, + {name: "retain legacy queued head", phase: core.PodPending, retained: true}, + {name: "replace succeeded head", phase: core.PodSucceeded}, + {name: "replace failed head", phase: core.PodFailed}, + {name: "delete auxiliary head with deployment", phase: core.PodRunning, teardown: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + f := newElasticConvergenceFixture(t) + pods := new(core.PodList) + require.NoError(t, f.reconciler.Client.List(ctx, pods, ctrlcli.InNamespace(f.md.Namespace))) + var head *core.Pod + for i := range pods.Items { + if modelDeploymentPodRole(&pods.Items[i]) == modelDeploymentElasticHeadName(f.md) { + head = pods.Items[i].DeepCopy() + break + } + } + require.NotNil(t, head) + if tc.retained { + head.Labels["kueue.x-k8s.io/queue-name"] = "legacy-cpu-queue" + head.Spec.Containers[0].Resources.Requests = core.ResourceList{core.ResourceCPU: resource.MustParse("1")} + require.NoError(t, f.reconciler.Client.Update(ctx, head)) + } + head.Status.Phase = tc.phase + require.NoError(t, f.reconciler.Client.Status().Update(ctx, head)) + if tc.teardown { + f.md.Finalizers = append(f.md.Finalizers, systemmeta.LockedResourceFinalizer) + require.NoError(t, f.reconciler.Client.Update(ctx, f.md)) + _, err := f.reconciler.teardownModelDeployment(ctx, f.md) + require.NoError(t, err) + } else { + _, err := f.reconciler.convergeModelDeployment(ctx, f.md) + require.NoError(t, err) + } + require.NoError(t, f.reconciler.Client.List(ctx, pods, ctrlcli.InNamespace(f.md.Namespace))) + heads := make([]core.Pod, 0, 1) + for _, pod := range pods.Items { + if modelDeploymentPodRole(&pod) == modelDeploymentElasticHeadName(f.md) { + heads = append(heads, pod) + } + } + if tc.teardown { + require.Empty(t, heads) + } else { + require.Len(t, heads, 1) + if tc.retained { + require.Equal(t, head.UID, heads[0].UID) + require.Equal(t, head.Spec, heads[0].Spec) + require.Equal(t, "legacy-cpu-queue", heads[0].Labels["kueue.x-k8s.io/queue-name"]) + } else { + require.NotEqual(t, head.Name, heads[0].Name) + require.NotContains(t, heads[0].Labels, "kueue.x-k8s.io/queue-name") + require.Empty(t, heads[0].Spec.Containers[0].Resources.Requests) + require.Empty(t, heads[0].Spec.Containers[0].Resources.Limits) + } + } + }) + } +} + func TestElasticWithdrawalWithoutRouterRetainsCapacity(t *testing.T) { ctx := context.Background() f := newElasticConvergenceFixture(t) @@ -664,7 +785,7 @@ func TestModelDeploymentElasticBootstrapFromMasterWidthBounds(t *testing.T) { md := newRenderDeployment(func(md *workercore.ModelDeployment) { md.Spec.Roles[0].ElasticEP = &workercore.ModelDeploymentRoleElasticEP{ - Width: tc.width, HeadInstanceType: "cpu", + Width: tc.width, } }) diff --git a/pkg/worker/controllers/worker/model_deployment_parallelism.go b/pkg/worker/controllers/worker/model_deployment_parallelism.go index 8f116b130..401c35ccd 100644 --- a/pkg/worker/controllers/worker/model_deployment_parallelism.go +++ b/pkg/worker/controllers/worker/model_deployment_parallelism.go @@ -29,6 +29,9 @@ type ModelDeploymentDeclaredParallelism struct { // sentinel for DP specified externally, the check reads it as undeclared, and the env // precedence still honors it. DataParallelLocal int + // ElasticEPMaxDataParallel is vLLM's startup DP capacity, not the running degree. + // Zero means no explicit --elastic-ep-max-dp-size declaration. + ElasticEPMaxDataParallel int // The degrees below have no key in today's transfer document; they are carried so the size // check and any future consumer read the full picture without a parser change. PrefillContextParallel int @@ -96,6 +99,7 @@ var modelDeploymentParallelFlags = map[string][]modelDeploymentParallelFlag{ {names: []string{"--pipeline-parallel-size", "-pp"}, degree: true, min: 1}, {names: []string{"--data-parallel-size", "-dp"}, degree: true, min: 1}, {names: []string{"--data-parallel-size-local", "-dpl"}, degree: true, min: 0}, + {names: []string{"--elastic-ep-max-dp-size"}, degree: true, min: 1}, {names: []string{"--prefill-context-parallel-size", "-pcp"}, abbr: []string{"-pc"}, degree: true, min: 1}, {names: []string{"--decode-context-parallel-size", "-dcp"}, degree: true, min: 1}, {names: []string{"--enable-expert-parallel", "-ep"}, abbr: []string{"-e"}, mode: true}, @@ -273,6 +277,8 @@ func scanModelDeploymentParallelism( declared.DataParallelLocal = degree reading.present[modelDeploymentSlotDataParallelLocal] = true localZeroDeclared = degree == 0 + case "--elastic-ep-max-dp-size": + declared.ElasticEPMaxDataParallel = degree case "--prefill-context-parallel-size": declared.PrefillContextParallel = degree reading.present[modelDeploymentSlotPrefillContextParallel] = true diff --git a/pkg/worker/controllers/worker/model_deployment_parallelism_test.go b/pkg/worker/controllers/worker/model_deployment_parallelism_test.go index 2fd8b42c4..06d73a24c 100644 --- a/pkg/worker/controllers/worker/model_deployment_parallelism_test.go +++ b/pkg/worker/controllers/worker/model_deployment_parallelism_test.go @@ -48,6 +48,12 @@ func TestParseModelDeploymentDeclaredParallelism(t *testing.T) { name: "vllm: empty books stay all ones", engine: vllm, }, + { + name: "vllm: startup capacity does not change running degrees", + engine: vllm, + extraArgs: []string{"--elastic-ep-max-dp-size=8"}, + set: func(p *ModelDeploymentDeclaredParallelism) { p.ElasticEPMaxDataParallel = 8 }, + }, { name: "vllm: long flag with separate value", engine: vllm, diff --git a/pkg/worker/controllers/worker/model_deployment_render.go b/pkg/worker/controllers/worker/model_deployment_render.go index a41a9b0db..263caf3ad 100644 --- a/pkg/worker/controllers/worker/model_deployment_render.go +++ b/pkg/worker/controllers/worker/model_deployment_render.go @@ -490,19 +490,7 @@ func renderModelDeploymentPodTemplate(ctx context.Context, in ModelDeploymentRen vols = append(vols, in.Connector.Volumes...) mounts = append(mounts, in.Connector.VolumeMounts...) } - if !slices.ContainsFunc(mounts, func(m core.VolumeMount) bool { return m.MountPath == "/dev/shm" }) { - size := resource.MustParse("16Gi") - if role.ShmSize != nil { - size = role.ShmSize.DeepCopy() - } - vols = append(vols, core.Volume{ - Name: "model-shm", - VolumeSource: core.VolumeSource{EmptyDir: &core.EmptyDirVolumeSource{ - Medium: core.StorageMediumMemory, SizeLimit: &size, - }}, - }) - mounts = append(mounts, core.VolumeMount{Name: "model-shm", MountPath: "/dev/shm"}) - } + vols, mounts = modelDeploymentSharedMemoryVolumes(role, vols, mounts) probePath := modelDeploymentProbePath probeScheme := scheme @@ -661,6 +649,27 @@ func renderModelDeploymentPodTemplate(ctx context.Context, in ModelDeploymentRen return pod, nil } +// modelDeploymentSharedMemoryVolumes adds the role's default mount unless one is explicit. +func modelDeploymentSharedMemoryVolumes( + role *workercore.ModelDeploymentRole, vols []core.Volume, mounts []core.VolumeMount, +) ([]core.Volume, []core.VolumeMount) { + if !slices.ContainsFunc(mounts, func(m core.VolumeMount) bool { return m.MountPath == "/dev/shm" }) { + size := resource.MustParse("16Gi") + if role.ShmSize != nil { + size = role.ShmSize.DeepCopy() + } + vols = append(vols, core.Volume{ + Name: "model-shm", + VolumeSource: core.VolumeSource{EmptyDir: &core.EmptyDirVolumeSource{ + Medium: core.StorageMediumMemory, SizeLimit: &size, + }}, + }) + mounts = append(mounts, core.VolumeMount{Name: "model-shm", MountPath: "/dev/shm"}) + } + + return vols, mounts +} + // modelDeploymentTCPTWReuseSysctl lets the kernel reuse a port held by a TIME-WAIT socket for a new // outgoing connection. It is namespaced, so it changes the Pod's network namespace and not the node's. const modelDeploymentTCPTWReuseSysctl = "net.ipv4.tcp_tw_reuse" diff --git a/pkg/worker/webhooks/worker/model_deployment_elastic.go b/pkg/worker/webhooks/worker/model_deployment_elastic.go index deef60453..5a77e601c 100644 --- a/pkg/worker/webhooks/worker/model_deployment_elastic.go +++ b/pkg/worker/webhooks/worker/model_deployment_elastic.go @@ -19,10 +19,8 @@ import ( "fmt" "strings" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" - ctrlcli "sigs.k8s.io/controller-runtime/pkg/client" workercore "gpustack.ai/gpustack/api/worker/v1alpha1" workerctrl "gpustack.ai/gpustack/pkg/worker/controllers/worker" @@ -82,7 +80,6 @@ func (r *ModelDeploymentWebhook) ValidateModelDeploymentElasticEP( if old != nil { errs = append(errs, validateModelDeploymentElasticIdentity(md, old)...) } - errs = append(errs, r.validateModelDeploymentElasticHeadInstanceType(ctx, md)...) return errs } @@ -164,18 +161,16 @@ func validateModelDeploymentElasticShape( if err != nil { errs = append(errs, field.Invalid(rolePath.Child("extraArgs"), role.ExtraArgs, err.Error())) } else { + if declared.ElasticEPMaxDataParallel > 0 && int64(role.ElasticEP.Width) > int64(declared.ElasticEPMaxDataParallel) { + errs = append(errs, field.Invalid(rolePath.Child("elasticEp", "width"), role.ElasticEP.Width, + fmt.Sprintf("must not exceed the startup --elastic-ep-max-dp-size (%d)", declared.ElasticEPMaxDataParallel))) + } if declared.PipelineParallel != 1 || declared.PrefillContextParallel != 1 { errs = append(errs, field.Invalid(rolePath.Child("extraArgs"), role.ExtraArgs, "each elastic member holds one TP group; pipeline and prefill context parallel sizes must be one")) } errs = append(errs, validateModelDeploymentElasticResources(role, rolePath, declared.TensorParallel)...) } - if role.ElasticEP.HeadInstanceType == "" { - errs = append(errs, field.Required( - rolePath.Child("elasticEp", "headInstanceType"), - "the Ray control-plane head needs a CPU-only InstanceType to run against", - )) - } if role.Kind != workercore.ModelDeploymentRoleKindServer { errs = append(errs, field.Invalid( rolePath.Child("kind"), role.Kind, @@ -312,18 +307,18 @@ func validateModelDeploymentElasticIdentity( return errs } - if oldElastic.ElasticEP.HeadInstanceType != newElastic.ElasticEP.HeadInstanceType { + oldParallel, oldErr := workerctrl.ParseModelDeploymentDeclaredParallelism(old.Spec.Engine.Name, oldElastic.ExtraArgs, oldElastic.Env) + newParallel, newErr := workerctrl.ParseModelDeploymentDeclaredParallelism(md.Spec.Engine.Name, newElastic.ExtraArgs, newElastic.Env) + if oldErr != nil || newErr != nil || oldParallel.TensorParallel != newParallel.TensorParallel { errs = append(errs, field.Invalid( - modelDeploymentElasticRolePath(md, newElastic).Child("elasticEp", "headInstanceType"), - newElastic.ElasticEP.HeadInstanceType, modelDeploymentIdentityMessage, + modelDeploymentElasticRolePath(md, newElastic).Child("extraArgs"), + newElastic.ExtraArgs, "tensor parallel size is fixed at creation; "+modelDeploymentIdentityMessage, )) } - oldTP, oldErr := workerctrl.ModelDeploymentElasticTensorParallelSize(oldElastic) - newTP, newErr := workerctrl.ModelDeploymentElasticTensorParallelSize(newElastic) - if oldErr != nil || newErr != nil || oldTP != newTP { - errs = append(errs, field.Invalid( + if oldErr == nil && newErr == nil && oldParallel.ElasticEPMaxDataParallel != newParallel.ElasticEPMaxDataParallel { + errs = append(errs, field.Forbidden( modelDeploymentElasticRolePath(md, newElastic).Child("extraArgs"), - newElastic.ExtraArgs, "tensor parallel size is fixed at creation; "+modelDeploymentIdentityMessage, + "--elastic-ep-max-dp-size is fixed at creation; "+modelDeploymentIdentityMessage, )) } if oldElastic.Name != newElastic.Name { @@ -341,37 +336,3 @@ func validateModelDeploymentElasticIdentity( return errs } - -// validateModelDeploymentElasticHeadInstanceType resolves each named head InstanceType and -// refuses anything acceleratable: the head is CPU-only by construction. -func (r *ModelDeploymentWebhook) validateModelDeploymentElasticHeadInstanceType( - ctx context.Context, md *workercore.ModelDeployment, -) field.ErrorList { - var errs field.ErrorList - - for i := range md.Spec.Roles { - role := &md.Spec.Roles[i] - if role.ElasticEP == nil || role.ElasticEP.HeadInstanceType == "" { - continue - } - headPath := field.NewPath("spec", "roles").Index(i).Child("elasticEp", "headInstanceType") - instType := &workercore.InstanceType{} - instType.Name = role.ElasticEP.HeadInstanceType - if err := r.APIReader.Get(ctx, ctrlcli.ObjectKeyFromObject(instType), instType); err != nil { - if apierrors.IsNotFound(err) { - errs = append(errs, field.NotFound(headPath, role.ElasticEP.HeadInstanceType)) - } else { - errs = append(errs, field.InternalError(headPath, fmt.Errorf("get instance type: %w", err))) - } - continue - } - if instType.Spec.Acceleratable { - errs = append(errs, field.Forbidden( - headPath, - "the Ray control-plane head runs CPU-only; an acceleratable InstanceType would put it on a GPU queue", - )) - } - } - - return errs -} diff --git a/pkg/worker/webhooks/worker/model_deployment_elastic_capacity_test.go b/pkg/worker/webhooks/worker/model_deployment_elastic_capacity_test.go new file mode 100644 index 000000000..ebbd26b9d --- /dev/null +++ b/pkg/worker/webhooks/worker/model_deployment_elastic_capacity_test.go @@ -0,0 +1,94 @@ +package worker + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + workercore "gpustack.ai/gpustack/api/worker/v1alpha1" +) + +func TestElasticMaxDPCapacityAdmission(t *testing.T) { + for _, tc := range []struct { + name string + args []string + width int32 + oldArgs []string + oldWidth int32 + wantErr string + wantField string + }{ + {name: "legacy omission", width: 2}, + {name: "legacy expansion", width: 4, oldWidth: 2}, + {name: "split maximum", args: []string{"--elastic-ep-max-dp-size", "4"}, width: 2}, + {name: "maximum equals width", args: []string{"--elastic-ep-max-dp-size=4"}, width: 4}, + {name: "larger startup maximum", args: []string{"--elastic-ep-max-dp-size=128"}, width: 4}, + {name: "create above maximum", args: []string{"--elastic-ep-max-dp-size=4"}, width: 5, wantErr: "--elastic-ep-max-dp-size", wantField: "elasticEp.width"}, + {name: "underscores", args: []string{"--elastic_ep_max_dp_size=4"}, width: 5, wantErr: "--elastic-ep-max-dp-size", wantField: "elasticEp.width"}, + {name: "unique prefix", args: []string{"--elastic-ep-max-dp", "4"}, width: 5, wantErr: "--elastic-ep-max-dp-size", wantField: "elasticEp.width"}, + {name: "last maximum wins", args: []string{"--elastic-ep-max-dp-size=2", "--elastic-ep-max-dp-size", "4"}, width: 4}, + {name: "last maximum refuses", args: []string{"--elastic-ep-max-dp-size=4", "--elastic-ep-max-dp-size=2"}, width: 4, wantErr: "--elastic-ep-max-dp-size", wantField: "elasticEp.width"}, + {name: "zero", args: []string{"--elastic-ep-max-dp-size=0"}, width: 2, wantErr: "below", wantField: "extraArgs"}, + {name: "negative", args: []string{"--elastic-ep-max-dp-size", "-1"}, width: 2, wantErr: "below", wantField: "extraArgs"}, + {name: "not integer", args: []string{"--elastic-ep-max-dp-size=4.5"}, width: 2, wantErr: "not an integer", wantField: "extraArgs"}, + {name: "overflow", args: []string{"--elastic-ep-max-dp-size=99999999999999999999999999"}, width: 2, wantErr: "out of range", wantField: "extraArgs"}, + {name: "missing value", args: []string{"--elastic-ep-max-dp-size"}, width: 2, wantErr: "has no value", wantField: "extraArgs"}, + {name: "end of options", args: []string{"--", "--elastic-ep-max-dp-size=2"}, width: 4}, + {name: "expand within maximum", args: []string{"--elastic-ep-max-dp-size=4"}, width: 4, oldArgs: []string{"--elastic-ep-max-dp-size", "4"}, oldWidth: 2}, + {name: "update above maximum", args: []string{"--elastic-ep-max-dp-size=4"}, width: 5, oldArgs: []string{"--elastic-ep-max-dp-size=4"}, oldWidth: 2, wantErr: "--elastic-ep-max-dp-size", wantField: "elasticEp.width"}, + {name: "increase maximum", args: []string{"--elastic-ep-max-dp-size=8"}, width: 4, oldArgs: []string{"--elastic-ep-max-dp-size=4"}, oldWidth: 2, wantErr: "fixed at creation", wantField: "extraArgs"}, + {name: "decrease maximum", args: []string{"--elastic-ep-max-dp-size=4"}, width: 2, oldArgs: []string{"--elastic-ep-max-dp-size=8"}, oldWidth: 2, wantErr: "fixed at creation", wantField: "extraArgs"}, + {name: "add maximum", args: []string{"--elastic-ep-max-dp-size=4"}, width: 2, oldWidth: 2, wantErr: "fixed at creation", wantField: "extraArgs"}, + {name: "remove maximum", width: 2, oldArgs: []string{"--elastic-ep-max-dp-size=4"}, oldWidth: 2, wantErr: "fixed at creation", wantField: "extraArgs"}, + {name: "equivalent maximum", args: []string{"--elastic_ep_max_dp_size", "+4"}, width: 4, oldArgs: []string{"--elastic-ep-max-dp-size=4"}, oldWidth: 2}, + {name: "unrelated tuning", args: []string{"--elastic-ep-max-dp-size=4", "--max-model-len=4096"}, width: 4, oldArgs: []string{"--elastic-ep-max-dp-size=4", "--max-model-len=2048"}, oldWidth: 2}, + } { + t.Run(tc.name, func(t *testing.T) { + md := elasticMD() + md.Spec.Roles[0].ExtraArgs = tc.args + md.Spec.Roles[0].ElasticEP.Width = tc.width + var old *workercore.ModelDeployment + if tc.oldWidth != 0 { + old = elasticMD() + old.Spec.Roles[0].ExtraArgs = tc.oldArgs + old.Spec.Roles[0].ElasticEP.Width = tc.oldWidth + } + w := new(ModelDeploymentWebhook) + errs := w.ValidateModelDeploymentElasticEP(context.Background(), md, old) + if tc.wantErr == "" { + assert.Empty(t, errs) + return + } + require.NotEmpty(t, errs) + assert.Equal(t, "spec.roles[0]."+tc.wantField, errs[0].Field) + assert.Contains(t, errs[0].Detail, tc.wantErr) + }) + } +} + +func TestElasticEPDiscardsLegacyHeadInstanceType(t *testing.T) { + for _, format := range []string{"json", "protobuf"} { + t.Run(format, func(t *testing.T) { + profile := new(workercore.ModelDeploymentRoleElasticEP) + if format == "json" { + require.NoError(t, json.Unmarshal([]byte(`{"width":2,"headInstanceType":"removed-cpu-type"}`), profile)) + } else { + legacy := make([]byte, 0, 4+len("removed-cpu-type")) + legacy = append(legacy, 0x08, 0x02, 0x12, byte(len("removed-cpu-type"))) + legacy = append(legacy, []byte("removed-cpu-type")...) + require.NoError(t, profile.Unmarshal(legacy)) + } + require.Equal(t, int32(2), profile.Width) + encoded, err := json.Marshal(profile) + require.NoError(t, err) + require.JSONEq(t, `{"width":2}`, string(encoded)) + md := elasticMD() + md.Spec.Roles[0].ElasticEP = profile + w := new(ModelDeploymentWebhook) + require.Empty(t, w.ValidateModelDeploymentElasticEP(context.Background(), md, nil)) + }) + } +} diff --git a/pkg/worker/webhooks/worker/model_deployment_elastic_test.go b/pkg/worker/webhooks/worker/model_deployment_elastic_test.go index a666ca656..dde30d194 100644 --- a/pkg/worker/webhooks/worker/model_deployment_elastic_test.go +++ b/pkg/worker/webhooks/worker/model_deployment_elastic_test.go @@ -22,7 +22,6 @@ import ( meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" - ctrlcli "sigs.k8s.io/controller-runtime/pkg/client" fakecli "sigs.k8s.io/controller-runtime/pkg/client/fake" workercore "gpustack.ai/gpustack/api/worker/v1alpha1" @@ -37,8 +36,7 @@ func elasticRole(mutate func(*workercore.ModelDeploymentRole)) workercore.ModelD ReplicaSize: 1, InstanceType: "nvidia-gpu", ElasticEP: &workercore.ModelDeploymentRoleElasticEP{ - Width: 2, - HeadInstanceType: "cpu-head", + Width: 2, }, } if mutate != nil { @@ -128,15 +126,6 @@ func TestValidateModelDeploymentElasticSpec(t *testing.T) { }, old: func() *workercore.ModelDeployment { return elasticMD() }, wantErr: []string{"elasticEp"}, - }, { - name: "headInstanceType change refused", - md: func() *workercore.ModelDeployment { - md := elasticMD() - md.Spec.Roles[0].ElasticEP.HeadInstanceType = "other-cpu" - return md - }, - old: func() *workercore.ModelDeployment { return elasticMD() }, - wantErr: []string{"headInstanceType"}, }, { name: "second elastic role refused", md: func() *workercore.ModelDeployment { @@ -263,13 +252,8 @@ func TestValidateModelDeploymentElasticSpec(t *testing.T) { })) }, }, { - name: "head instance type not found refused", - md: func() *workercore.ModelDeployment { return elasticMD() }, - wantErr: []string{"headInstanceType"}, - }, { - name: "head instance type acceleratable refused", - md: func() *workercore.ModelDeployment { return elasticMD() }, - wantErr: []string{"headInstanceType"}, + name: "no CPU head instance type required", + md: func() *workercore.ModelDeployment { return elasticMD() }, }} for _, tc := range testCases { @@ -280,15 +264,7 @@ func TestValidateModelDeploymentElasticSpec(t *testing.T) { if err := workercore.AddToScheme(scheme); err != nil { t.Fatalf("build scheme: %v", err) } - objs := []ctrlcli.Object{} - if tc.name != "head instance type not found refused" { - acceleratable := tc.name == "head instance type acceleratable refused" - objs = append(objs, &workercore.InstanceType{ - ObjectMeta: meta.ObjectMeta{Name: "cpu-head"}, - Spec: workercore.InstanceTypeSpec{Acceleratable: acceleratable}, - }) - } - cli := fakecli.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build() + cli := fakecli.NewClientBuilder().WithScheme(scheme).Build() webhook := &ModelDeploymentWebhook{APIReader: cli} var old *workercore.ModelDeployment diff --git a/specs/2026-10-01-s2-elastic-ep-resize.md b/specs/2026-10-01-s2-elastic-ep-resize.md index 7ff0cc88d..b4be8d204 100644 --- a/specs/2026-10-01-s2-elastic-ep-resize.md +++ b/specs/2026-10-01-s2-elastic-ep-resize.md @@ -233,32 +233,49 @@ separation of continuity claims from instance-interruption characterization. ### Elastic request contract -The managed profile adds optional `roles[].elasticEp` with two required fields: +The managed profile adds optional `roles[].elasticEp` with one required field: | Field | Meaning | Update rule | | --- | --- | --- | | `width` | Total GPU engines, including the reserved API/DP-master; integer from 2 through 64. | Mutable. | -| `headInstanceType` | Name of an existing CPU-only InstanceType for the Ray head. | Immutable. | Profile presence is immutable. The deployment has one managed vLLM Server role, with `replicas=1` and `size=1`. Those fields retain their existing meanings. This profile supports the pinned vLLM release with PP=1 and prefill context parallel size one. TP is read from `extraArgs` and stays fixed while DP width changes. +An explicit `--elastic-ep-max-dp-size` in `extraArgs` bounds the target width on create and update. +The effective maximum is fixed at creation; adding, removing or changing it requires a new deployment. +Equivalent argument forms are accepted. The operator does not add a duplicate API field or inject the flag. +The argument is available starting with vLLM v0.30.0: +[CLI definition](https://github.com/vllm-project/vllm/blob/ced6857afa0ea7b2e3f0846a62e1394e90f15607/vllm/engine/arg_utils.py#L1213-L1216). +Omission preserves legacy admission. In vLLM v0.31.0, the omitted maximum equals initial DP, +so deployments using that version must set a larger maximum before startup to expand online. +This version contract is source-checked, not GPU-verified: +[configuration](https://github.com/vllm-project/vllm/blob/db9527a46873454610df6dbedf79a36d6bf1a7f6/vllm/config/parallel.py#L971-L981), +[resize admission](https://github.com/vllm-project/vllm/blob/db9527a46873454610df6dbedf79a36d6bf1a7f6/vllm/v1/engine/core_client.py#L1746-L1762). Each member requests the whole GPU count of its TP group; omitted TP defaults to one. Command takeover, slicing, partitioning and fabric interface requests are refused. Arguments or environment values that override operator-owned elastic settings are refused. -The head uses its CPU InstanceType's unit resources and independent CPU queue admission. +The head is a directly managed auxiliary Pod with no InstanceType or LocalQueue dependency. +Its template declares no container or Pod resource requests or limits, including CPU, memory and GPU. +Namespace admission can inject defaults; the controller does not add them. It consumes no GPU width. Only the reserved master supplies API endpoints and serving status. Initial boot width is retained separately from the mutable target. A target or deployment annotation change cannot replace a serving master. A smaller target alone cannot delete workers. This request contract does not freeze a public effective-width status field or prove native support. + ### Feature 1 — Elastic workload realization with four-layer resource reconciliation (E9 / T07) **Shape.** One ModelDeployment instance maps to one dedicated logical Ray cluster. The operator creates and owns its head and worker Pods. The Ray head is CPU-only -and carries only the Ray control plane. Each worker Pod holds one complete TP group with +and carries only the Ray control plane. It starts with `--num-gpus=0`. +GPU master and worker commands omit `--num-gpus` so Ray discovers container-visible devices. +GPU requests, device allocation and TP remain unchanged. +The existing observer rejects any live GPU Ray node whose logical GPU capacity differs from TP. +Logical Ray capacity does not prove actual device allocation or Pod identity. +Auto-discovery must also be verified in the pinned runtime; no slicing or multi-vendor support is inferred. Each worker Pod holds one complete TP group with `numOfHosts=1`. The vLLM API/DP-master role is pinned to a reserved GPU worker Pod with explicit `data_parallel_size_local=1` in Internal mode (strict placement requires a positive local size on a GPU-bearing node; a CPU head hosting the API master with strict placement and `local=0` allocates @@ -324,7 +341,7 @@ whole-GPU admission only; Ray/elastic realization **NOT-established**). **MEASURED** — the pinned native profile failed before inference: its `/dev/shm` buffer required 160 MiB, with 64 MiB available. Ordinary role rendering provides each Pod with a memory-backed `EmptyDir` at `/dev/shm`. `roles[].shmSize` is an optional positive Kubernetes quantity. Omission renders `16Gi`. -The common path covers ordinary roles, take-over roles, Elastic GPU members and the CPU head. +Ordinary roles, take-over roles, Elastic GPU members and the auxiliary CPU head share this volume policy. The volume enters the Pod spec hash. Its capacity does not reserve RAM or raise the memory request. Only the main container mounts it; used shared memory counts toward that container's memory limit. An explicit `/dev/shm` mount takes precedence; its capacity and backing remain the user's responsibility.