Skip to content

feat(instance): support proxy pod scheduling #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vitaliytv wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(instance): support proxy pod scheduling #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions api/v1alpha1/litellminstance_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,11 @@ type LiteLLMInstanceSpec struct {
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Deployment"
Deployment *DeploymentSpec `json:"deployment,omitempty"`

// PodScheduling configures scheduling for LiteLLM proxy Pods.
// +optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Pod Scheduling"
PodScheduling *PodSchedulingSpec `json:"podScheduling,omitempty"`

// Ingress configuration.
// +optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Ingress"
Expand Down Expand Up @@ -1164,6 +1169,17 @@ type DeploymentSpec struct {
Annotations map[string]string `json:"annotations,omitempty"`
}

// PodSchedulingSpec configures node placement for LiteLLM proxy Pods.
type PodSchedulingSpec struct {
// NodeSelector selects the nodes where LiteLLM proxy Pods may run.
// +optional
NodeSelector map[string]string `json:"nodeSelector,omitempty"`

// Tolerations allows LiteLLM proxy Pods to run on matching tainted nodes.
// +optional
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
}

// IngressSpec defines Ingress configuration.
type IngressSpec struct {
// Enable Ingress.
Expand Down
34 changes: 34 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

51 changes: 51 additions & 0 deletions config/crd/bases/litellm.palena.ai_litellminstances.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3571,6 +3571,57 @@ spec:
required:
- enabled
type: object
podScheduling:
description: PodScheduling configures scheduling for LiteLLM proxy
Pods.
properties:
nodeSelector:
additionalProperties:
type: string
description: NodeSelector selects the nodes where LiteLLM proxy
Pods may run.
type: object
tolerations:
description: Tolerations allows LiteLLM proxy Pods to run on matching
tainted nodes.
items:
description: |-
The pod this Toleration is attached to tolerates any taint that matches
the triple <key,value,effect> using the matching operator <operator>.
properties:
effect:
description: |-
Effect indicates the taint effect to match. Empty means match all taint effects.
When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
type: string
key:
description: |-
Key is the taint key that the toleration applies to. Empty means match all taint keys.
If the key is empty, operator must be Exists; this combination means to match all values and all keys.
type: string
operator:
description: |-
Operator represents a key's relationship to the value.
Valid operators are Exists and Equal. Defaults to Equal.
Exists is equivalent to wildcard for value, so that a pod can
tolerate all taints of a particular category.
type: string
tolerationSeconds:
description: |-
TolerationSeconds represents the period of time the toleration (which must be
of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
it is not set, which means tolerate the taint forever (do not evict). Zero and
negative values will be treated as 0 (evict immediately) by the system.
format: int64
type: integer
value:
description: |-
Value is the taint value the toleration matches to.
If the operator is Exists, the value should be empty, otherwise just a regular string.
type: string
type: object
type: array
type: object
rbac:
description: |-
Role-based access control configuration.
Expand Down
51 changes: 51 additions & 0 deletions deploy/charts/litellm-operator/crds/litellm.palena.ai_litellminstances.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3571,6 +3571,57 @@ spec:
required:
- enabled
type: object
podScheduling:
description: PodScheduling configures scheduling for LiteLLM proxy
Pods.
properties:
nodeSelector:
additionalProperties:
type: string
description: NodeSelector selects the nodes where LiteLLM proxy
Pods may run.
type: object
tolerations:
description: Tolerations allows LiteLLM proxy Pods to run on matching
tainted nodes.
items:
description: |-
The pod this Toleration is attached to tolerates any taint that matches
the triple <key,value,effect> using the matching operator <operator>.
properties:
effect:
description: |-
Effect indicates the taint effect to match. Empty means match all taint effects.
When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
type: string
key:
description: |-
Key is the taint key that the toleration applies to. Empty means match all taint keys.
If the key is empty, operator must be Exists; this combination means to match all values and all keys.
type: string
operator:
description: |-
Operator represents a key's relationship to the value.
Valid operators are Exists and Equal. Defaults to Equal.
Exists is equivalent to wildcard for value, so that a pod can
tolerate all taints of a particular category.
type: string
tolerationSeconds:
description: |-
TolerationSeconds represents the period of time the toleration (which must be
of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
it is not set, which means tolerate the taint forever (do not evict). Zero and
negative values will be treated as 0 (evict immediately) by the system.
format: int64
type: integer
value:
description: |-
Value is the taint value the toleration matches to.
If the operator is Exists, the value should be empty, otherwise just a regular string.
type: string
type: object
type: array
type: object
rbac:
description: |-
Role-based access control configuration.
Expand Down
5 changes: 5 additions & 0 deletions internal/resources/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package resources
import (
"fmt"
"maps"
"slices"
"strings"

appsv1 "k8s.io/api/apps/v1"
Expand Down Expand Up @@ -278,6 +279,10 @@ func BuildDeployment(instance *litellmv1alpha1.LiteLLMInstance, labels map[strin
if len(instance.Spec.TopologySpreadConstraints) > 0 {
dep.Spec.Template.Spec.TopologySpreadConstraints = instance.Spec.TopologySpreadConstraints
}
if scheduling := instance.Spec.PodScheduling; scheduling != nil {
dep.Spec.Template.Spec.NodeSelector = maps.Clone(scheduling.NodeSelector)
dep.Spec.Template.Spec.Tolerations = slices.Clone(scheduling.Tolerations)
}

return dep
}
Expand Down
21 changes: 21 additions & 0 deletions internal/resources/deployment_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,27 @@ func TestBuildDeployment_NoDeploymentAnnotationsByDefault(t *testing.T) {
}
}

func TestBuildDeployment_PodScheduling(t *testing.T) {
instance := newTestInstance()
instance.Spec.PodScheduling = &litellmv1alpha1.PodSchedulingSpec{
NodeSelector: map[string]string{"kubernetes.io/arch": "arm64"},
Tolerations: []corev1.Toleration{{
Key: "kubernetes.io/arch",
Operator: corev1.TolerationOpEqual,
Value: "arm64",
Effect: corev1.TaintEffectNoSchedule,
}},
}

podSpec := BuildDeployment(instance, map[string]string{"app": "litellm"}, "", nil).Spec.Template.Spec
if got := podSpec.NodeSelector["kubernetes.io/arch"]; got != "arm64" {
t.Errorf("expected arm64 node selector, got %q", got)
}
if len(podSpec.Tolerations) != 1 || podSpec.Tolerations[0].Effect != corev1.TaintEffectNoSchedule {
t.Errorf("expected configured toleration, got %#v", podSpec.Tolerations)
}
}

func TestBuildDeployment_LicenseSecretChangesTemplate(t *testing.T) {
instance := newTestInstance()
labels := map[string]string{"app": "litellm"}
Expand Down
Loading