diff --git a/api/v1alpha1/litellminstance_types.go b/api/v1alpha1/litellminstance_types.go index b6dc093..227853b 100644 --- a/api/v1alpha1/litellminstance_types.go +++ b/api/v1alpha1/litellminstance_types.go @@ -95,6 +95,11 @@ type LiteLLMInstanceSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Deployment" Deployment *DeploymentSpec `json:"deployment,omitempty"` + // PodScheduling configures scheduling for LiteLLM proxy Pods. + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Pod Scheduling" + PodScheduling *PodSchedulingSpec `json:"podScheduling,omitempty"` + // Ingress configuration. // +optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Ingress" @@ -1164,6 +1169,17 @@ type DeploymentSpec struct { Annotations map[string]string `json:"annotations,omitempty"` } +// PodSchedulingSpec configures node placement for LiteLLM proxy Pods. +type PodSchedulingSpec struct { + // NodeSelector selects the nodes where LiteLLM proxy Pods may run. + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // Tolerations allows LiteLLM proxy Pods to run on matching tainted nodes. + // +optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` +} + // IngressSpec defines Ingress configuration. type IngressSpec struct { // Enable Ingress. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index c0ec1b7..71118ba 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -2012,6 +2012,11 @@ func (in *LiteLLMInstanceSpec) DeepCopyInto(out *LiteLLMInstanceSpec) { *out = new(DeploymentSpec) (*in).DeepCopyInto(*out) } + if in.PodScheduling != nil { + in, out := &in.PodScheduling, &out.PodScheduling + *out = new(PodSchedulingSpec) + (*in).DeepCopyInto(*out) + } if in.Ingress != nil { in, out := &in.Ingress, &out.Ingress *out = new(IngressSpec) @@ -3694,6 +3699,35 @@ func (in *PersonalKeyGenerationSettings) DeepCopy() *PersonalKeyGenerationSettin return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodSchedulingSpec) DeepCopyInto(out *PodSchedulingSpec) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodSchedulingSpec. +func (in *PodSchedulingSpec) DeepCopy() *PodSchedulingSpec { + if in == nil { + return nil + } + out := new(PodSchedulingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PrometheusRuleSpec) DeepCopyInto(out *PrometheusRuleSpec) { *out = *in diff --git a/config/crd/bases/litellm.palena.ai_litellminstances.yaml b/config/crd/bases/litellm.palena.ai_litellminstances.yaml index 507ddfc..e953130 100644 --- a/config/crd/bases/litellm.palena.ai_litellminstances.yaml +++ b/config/crd/bases/litellm.palena.ai_litellminstances.yaml @@ -3571,6 +3571,57 @@ spec: required: - enabled type: object + podScheduling: + description: PodScheduling configures scheduling for LiteLLM proxy + Pods. + properties: + nodeSelector: + additionalProperties: + type: string + description: NodeSelector selects the nodes where LiteLLM proxy + Pods may run. + type: object + tolerations: + description: Tolerations allows LiteLLM proxy Pods to run on matching + tainted nodes. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object rbac: description: |- Role-based access control configuration. diff --git a/deploy/charts/litellm-operator/crds/litellm.palena.ai_litellminstances.yaml b/deploy/charts/litellm-operator/crds/litellm.palena.ai_litellminstances.yaml index 507ddfc..e953130 100644 --- a/deploy/charts/litellm-operator/crds/litellm.palena.ai_litellminstances.yaml +++ b/deploy/charts/litellm-operator/crds/litellm.palena.ai_litellminstances.yaml @@ -3571,6 +3571,57 @@ spec: required: - enabled type: object + podScheduling: + description: PodScheduling configures scheduling for LiteLLM proxy + Pods. + properties: + nodeSelector: + additionalProperties: + type: string + description: NodeSelector selects the nodes where LiteLLM proxy + Pods may run. + type: object + tolerations: + description: Tolerations allows LiteLLM proxy Pods to run on matching + tainted nodes. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object rbac: description: |- Role-based access control configuration. diff --git a/internal/resources/deployment.go b/internal/resources/deployment.go index 02ebcc6..a17b0a9 100644 --- a/internal/resources/deployment.go +++ b/internal/resources/deployment.go @@ -19,6 +19,7 @@ package resources import ( "fmt" "maps" + "slices" "strings" appsv1 "k8s.io/api/apps/v1" @@ -278,6 +279,10 @@ func BuildDeployment(instance *litellmv1alpha1.LiteLLMInstance, labels map[strin if len(instance.Spec.TopologySpreadConstraints) > 0 { dep.Spec.Template.Spec.TopologySpreadConstraints = instance.Spec.TopologySpreadConstraints } + if scheduling := instance.Spec.PodScheduling; scheduling != nil { + dep.Spec.Template.Spec.NodeSelector = maps.Clone(scheduling.NodeSelector) + dep.Spec.Template.Spec.Tolerations = slices.Clone(scheduling.Tolerations) + } return dep } diff --git a/internal/resources/deployment_test.go b/internal/resources/deployment_test.go index 23b7178..cd9f3e4 100644 --- a/internal/resources/deployment_test.go +++ b/internal/resources/deployment_test.go @@ -113,6 +113,27 @@ func TestBuildDeployment_NoDeploymentAnnotationsByDefault(t *testing.T) { } } +func TestBuildDeployment_PodScheduling(t *testing.T) { + instance := newTestInstance() + instance.Spec.PodScheduling = &litellmv1alpha1.PodSchedulingSpec{ + NodeSelector: map[string]string{"kubernetes.io/arch": "arm64"}, + Tolerations: []corev1.Toleration{{ + Key: "kubernetes.io/arch", + Operator: corev1.TolerationOpEqual, + Value: "arm64", + Effect: corev1.TaintEffectNoSchedule, + }}, + } + + podSpec := BuildDeployment(instance, map[string]string{"app": "litellm"}, "", nil).Spec.Template.Spec + if got := podSpec.NodeSelector["kubernetes.io/arch"]; got != "arm64" { + t.Errorf("expected arm64 node selector, got %q", got) + } + if len(podSpec.Tolerations) != 1 || podSpec.Tolerations[0].Effect != corev1.TaintEffectNoSchedule { + t.Errorf("expected configured toleration, got %#v", podSpec.Tolerations) + } +} + func TestBuildDeployment_LicenseSecretChangesTemplate(t *testing.T) { instance := newTestInstance() labels := map[string]string{"app": "litellm"}