Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions api/v1alpha2/task_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,15 @@ type PodOverrides struct {
// +optional
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`

// RuntimeClassName selects the RuntimeClass for the agent pod, such as
// a gVisor or Kata Containers sandbox. The RuntimeClass must exist in
// the cluster, or the pod is rejected at admission.
// +optional
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=253
// +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`
RuntimeClassName *string `json:"runtimeClassName,omitempty"`

// ServiceAccountName sets the pod's service account.
// Use with workload identity systems such as IRSA on EKS, GKE
// Workload Identity, or Azure Workload Identity.
Expand Down
5 changes: 5 additions & 0 deletions api/v1alpha2/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 6 additions & 2 deletions docs/reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ Exactly one execution source is required: `spec.worker` (preferred), `spec.worke
| `spec.podOverrides.tolerations` | Tolerations for the agent pod; use with `nodeSelector` or `affinity` to target dedicated node pools (e.g., GPU nodes, agent-specific pools) | No |
| `spec.podOverrides.affinity` | Node, pod, and pod-anti-affinity rules. Use for spreading agents across nodes or expressing scheduling preferences beyond `nodeSelector` | No |
| `spec.podOverrides.imagePullSecrets` | Secrets used to pull container images from private registries. Required when the agent image or any init container image is in a private registry | No |
| `spec.podOverrides.runtimeClassName` | RuntimeClass for the agent pod, e.g. a gVisor or Kata Containers sandbox. Must be a DNS subdomain name. The RuntimeClass must already exist in the cluster; otherwise Pod creation fails and the Job records `FailedCreate` events | No |
| `spec.podOverrides.serviceAccountName` | Service account name for the agent pod; use with workload identity systems (IRSA, GKE Workload Identity, Azure) | No |
| `spec.podOverrides.volumes` | Additional volumes to attach to the agent pod. Names must not be `workspace` or use the Kelos-reserved `kelos-` prefix | No |
| `spec.podOverrides.volumeMounts` | Additional volume mounts on the agent container; names must reference either a user-supplied volume from `volumes` or a Kelos-managed volume (`workspace` or a `kelos-` volume such as `kelos-plugin` or `kelos-github-token`) | No |
Expand Down Expand Up @@ -317,6 +318,7 @@ the Session resource and is visible through the Kubernetes API.
| `spec.worker.agentConfigRefs[].name` | Ordered AgentConfig resources | No |
| `spec.worker.podOverrides` | Pod resources, scheduling, environment, volumes, and sidecars | No |
| `spec.worker.podOverrides.serviceAccountName` | Service account for the Session Pod; omit or clear to use the Session's managed service account | No |
| `spec.worker.podOverrides.runtimeClassName` | RuntimeClass for the Session Pod, e.g. a gVisor or Kata Containers sandbox. The RuntimeClass must already exist in the cluster; otherwise Pod creation fails and the StatefulSet records `FailedCreate` events. The Session's own agent can change or clear it (see below) | No |
| `spec.suspend` | Stop the Session runtime without deleting the Session or its persistent workspace (defaults to `false`) | No |
| `spec.initialBranch` | Git branch used to initialize the Session workspace. Checks out the branch from `origin` when it exists, or creates it from the Workspace ref. Requires `spec.worker.workspaceRef` | No |
| `spec.initialPrompt` | Prompt submitted when the Session starts without retained conversation history. An `emptyDir` workspace may submit it again after Pod replacement | No |
Expand Down Expand Up @@ -597,8 +599,10 @@ change its own `spec.worker.podOverrides.serviceAccountName` to another existing
account in the same namespace. This can grant that account's Kubernetes
permissions and any configured workload identity. Kelos does not restrict this
change to operators: Session agents must be trusted to select service accounts
in their namespace. Deployments that require restrictions must enforce them
through an administrator-managed admission policy.
in their namespace. The same applies to `spec.worker.podOverrides.runtimeClassName`:
the agent can clear it or pick another RuntimeClass, and the replacement Pod
runs without the original sandbox. Deployments that require restrictions must
enforce them through an administrator-managed admission policy.

Some nested `spec.worker.podOverrides` field descriptions inherit Kubernetes
wording such as "Cannot be updated". Those restrictions apply to editing an
Expand Down
6 changes: 6 additions & 0 deletions internal/controller/job_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -803,6 +803,7 @@ fi`,
var tolerations []corev1.Toleration
var affinity *corev1.Affinity
var imagePullSecrets []corev1.LocalObjectReference
var runtimeClassName *string

if po := resolveTaskPodOverrides(task); po != nil {
if po.Labels != nil {
Expand Down Expand Up @@ -855,6 +856,10 @@ fi`,
imagePullSecrets = po.ImagePullSecrets
}

if po.RuntimeClassName != nil {
runtimeClassName = po.RuntimeClassName
}

if po.ServiceAccountName != "" {
serviceAccountName = po.ServiceAccountName
}
Expand Down Expand Up @@ -963,6 +968,7 @@ fi`,
Tolerations: tolerations,
Affinity: affinity,
ImagePullSecrets: imagePullSecrets,
RuntimeClassName: runtimeClassName,
},
},
},
Expand Down
36 changes: 36 additions & 0 deletions internal/controller/job_builder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2466,6 +2466,37 @@ func TestBuildJob_PodOverridesImagePullSecrets(t *testing.T) {
}
}

func TestBuildJob_PodOverridesRuntimeClassName(t *testing.T) {
builder := NewJobBuilder()
task := &kelos.Task{
ObjectMeta: metav1.ObjectMeta{
Name: "test-runtime-class",
Namespace: "default",
},
Spec: kelos.TaskSpec{
Type: AgentTypeClaudeCode,
Prompt: "Fix issue",
Credentials: &kelos.Credentials{
Type: kelos.CredentialTypeAPIKey,
SecretRef: &kelos.SecretReference{Name: "my-secret"},
},
PodOverrides: &kelos.PodOverrides{
RuntimeClassName: ptr.To("gvisor"),
},
},
}

job, err := builder.Build(task, nil, nil, task.Spec.Prompt)
if err != nil {
t.Fatalf("Build() returned error: %v", err)
}

got := job.Spec.Template.Spec.RuntimeClassName
if got == nil || *got != "gvisor" {
t.Errorf("RuntimeClassName = %v, want %q", got, "gvisor")
}
}

func TestBuildJob_PodOverridesAllFields(t *testing.T) {
builder := NewJobBuilder()
task := &kelos.Task{
Expand Down Expand Up @@ -2724,6 +2755,11 @@ func TestBuildJob_NoPodOverrides(t *testing.T) {
if job.Spec.Template.Spec.ImagePullSecrets != nil {
t.Error("Expected no ImagePullSecrets when PodOverrides is nil")
}

// No RuntimeClassName.
if job.Spec.Template.Spec.RuntimeClassName != nil {
t.Error("Expected no RuntimeClassName when PodOverrides is nil")
}
}

func TestBuildJob_AgentConfigAgentsMD(t *testing.T) {
Expand Down
15 changes: 15 additions & 0 deletions internal/controller/session_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1715,6 +1715,21 @@ func TestSessionRuntimeUsesConfiguredServiceAccount(t *testing.T) {
}
}

func TestSessionRuntimeUsesConfiguredRuntimeClassName(t *testing.T) {
t.Parallel()
session := testSession("custom-runtime-class", "codex")
session.Spec.Worker.PodOverrides = &kelos.PodOverrides{RuntimeClassName: ptr.To("gvisor")}
reconciler := testSessionReconciler(nil, nil)
statefulSet, _, err := reconciler.buildSessionStatefulSet(session, nil, nil)
if err != nil {
t.Fatal(err)
}
got := statefulSet.Spec.Template.Spec.RuntimeClassName
if got == nil || *got != "gvisor" {
t.Fatalf("Session runtimeClassName = %v, want gvisor", got)
}
}

func TestPrepareSessionWorkspaceInitPreservesCloneCommands(t *testing.T) {
t.Parallel()
tests := []struct {
Expand Down
9 changes: 8 additions & 1 deletion internal/controller/workerpool_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -943,6 +943,7 @@ func (r *WorkerPoolReconciler) buildStatefulSet(pool *kelos.WorkerPool, stsName,
var tolerations []corev1.Toleration
var affinity *corev1.Affinity
var imagePullSecrets []corev1.LocalObjectReference
var runtimeClassName *string

if po := pool.Spec.Worker.PodOverrides; po != nil {
if po.Resources != nil {
Expand Down Expand Up @@ -981,6 +982,10 @@ func (r *WorkerPoolReconciler) buildStatefulSet(pool *kelos.WorkerPool, stsName,
imagePullSecrets = po.ImagePullSecrets
}

if po.RuntimeClassName != nil {
runtimeClassName = po.RuntimeClassName
}

if len(po.Volumes) > 0 {
if err := validateUserVolumes(po.Volumes); err != nil {
return nil, err
Expand Down Expand Up @@ -1057,6 +1062,7 @@ func (r *WorkerPoolReconciler) buildStatefulSet(pool *kelos.WorkerPool, stsName,
Tolerations: tolerations,
Affinity: affinity,
ImagePullSecrets: imagePullSecrets,
RuntimeClassName: runtimeClassName,
},
},
VolumeClaimTemplates: []corev1.PersistentVolumeClaim{
Expand Down Expand Up @@ -1886,7 +1892,8 @@ func podTemplateSpecEqual(a, b corev1.PodSpec) bool {
apiequality.Semantic.DeepEqual(a.NodeSelector, b.NodeSelector) &&
apiequality.Semantic.DeepEqual(a.Tolerations, b.Tolerations) &&
apiequality.Semantic.DeepEqual(a.Affinity, b.Affinity) &&
apiequality.Semantic.DeepEqual(a.ImagePullSecrets, b.ImagePullSecrets)
apiequality.Semantic.DeepEqual(a.ImagePullSecrets, b.ImagePullSecrets) &&
apiequality.Semantic.DeepEqual(a.RuntimeClassName, b.RuntimeClassName)
}

func mergeReconcileResults(a, b ctrl.Result) ctrl.Result {
Expand Down
36 changes: 36 additions & 0 deletions internal/controller/workerpool_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/record"
"k8s.io/utils/ptr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
Expand Down Expand Up @@ -328,6 +329,41 @@ func TestWorkerPoolReconciler_RejectsExtraContainerInitContainerNameCollision(t
assert.Contains(t, err.Error(), "appears in both extraContainers and extraInitContainers")
}

func TestWorkerPoolReconciler_UpdatesRuntimeClassName(t *testing.T) {
scheme := newWorkerPoolTestScheme()
pool := newTestWorkerPool("my-pool", "default", 1)
ws := newTestWorkspace("default")

cl := fake.NewClientBuilder().
WithScheme(scheme).
WithStatusSubresource(&kelos.WorkerPool{}).
WithObjects(pool, ws).
Build()

r := newWorkerPoolReconciler(cl, scheme)
req := ctrl.Request{NamespacedName: types.NamespacedName{Name: "my-pool", Namespace: "default"}}
stsKey := types.NamespacedName{Name: "wp-my-pool", Namespace: "default"}

_, err := r.Reconcile(context.Background(), req)
require.NoError(t, err)

var sts appsv1.StatefulSet
require.NoError(t, cl.Get(context.Background(), stsKey, &sts))
assert.Nil(t, sts.Spec.Template.Spec.RuntimeClassName)

// Setting the field on a running pool must roll the StatefulSet.
require.NoError(t, cl.Get(context.Background(), req.NamespacedName, pool))
pool.Spec.Worker.PodOverrides = &kelos.PodOverrides{RuntimeClassName: ptr.To("gvisor")}
require.NoError(t, cl.Update(context.Background(), pool))

_, err = r.Reconcile(context.Background(), req)
require.NoError(t, err)

require.NoError(t, cl.Get(context.Background(), stsKey, &sts))
require.NotNil(t, sts.Spec.Template.Spec.RuntimeClassName)
assert.Equal(t, "gvisor", *sts.Spec.Template.Spec.RuntimeClassName)
}

func TestWorkerPoolReconciler_CreatesService(t *testing.T) {
scheme := newWorkerPoolTestScheme()
pool := newTestWorkerPool("my-pool", "default", 2)
Expand Down
44 changes: 44 additions & 0 deletions internal/conversion/taskspawner.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"strings"
"unicode/utf8"

"k8s.io/apimachinery/pkg/util/validation"

v1alpha1 "github.com/kelos-dev/kelos/api/v1alpha1"
v1alpha2 "github.com/kelos-dev/kelos/api/v1alpha2"
)
Expand Down Expand Up @@ -62,6 +64,11 @@ type preservedWebhookGatewayRefs struct {
// why the value is preserved rather than left to fall away.
const preservedSlackExcludeFiltersAnnotation = "kelos.dev/v1alpha2-slack-exclude-filters"

// preservedRuntimeClassNameAnnotation carries the task template's
// podOverrides.runtimeClassName across a v1alpha1 round-trip. Dropping it
// would run every later Task outside the sandbox it selects.
const preservedRuntimeClassNameAnnotation = "kelos.dev/v1alpha2-runtime-class-name"

var slackFilterChannelIDPattern = regexp.MustCompile(v1alpha2.SlackFilterChannelIDPattern)

type preservedGitHubCommentsReporting struct {
Expand Down Expand Up @@ -102,6 +109,8 @@ func taskSpawnerToHub(_ context.Context, src *v1alpha1.TaskSpawner, dst *v1alpha
deleteAnnotation(dst.Annotations, preservedGitHubWebhookExcludeFiltersAnnotation)
restorePreservedSlackExcludeFilters(src.Annotations, dst.Spec.When.Slack)
deleteAnnotation(dst.Annotations, preservedSlackExcludeFiltersAnnotation)
restorePreservedRuntimeClassName(src.Annotations, &dst.Spec.TaskTemplate)
deleteAnnotation(dst.Annotations, preservedRuntimeClassNameAnnotation)
return nil
}

Expand Down Expand Up @@ -133,9 +142,44 @@ func taskSpawnerFromHub(_ context.Context, src *v1alpha2.TaskSpawner, dst *v1alp
if err := setPreservedSlackExcludeFilters(dst, src.Spec.When.Slack); err != nil {
return err
}
setPreservedRuntimeClassName(dst, src.Spec.TaskTemplate)
return convertViaJSON(&src.Status, &dst.Status)
}

// setPreservedRuntimeClassName records the runtimeClassName of the pod
// overrides that backfillTaskTemplateLegacyWorkerFields copies to v1alpha1:
// the legacy field when set, otherwise the worker's.
func setPreservedRuntimeClassName(dst *v1alpha1.TaskSpawner, template v1alpha2.TaskTemplate) {
po := template.PodOverrides
if po == nil && template.WorkerPoolRef == nil && template.Worker != nil {
po = template.Worker.PodOverrides
}
if po == nil || po.RuntimeClassName == nil {
deleteAnnotation(dst.Annotations, preservedRuntimeClassNameAnnotation)
return
}
if dst.Annotations == nil {
dst.Annotations = map[string]string{}
}
dst.Annotations[preservedRuntimeClassNameAnnotation] = *po.RuntimeClassName
}

// restorePreservedRuntimeClassName puts the value back on the legacy pod
// overrides, which is where the v1alpha1 pod overrides land in the hub.
// Values the v1alpha2 schema would reject are ignored.
func restorePreservedRuntimeClassName(annotations map[string]string, dst *v1alpha2.TaskTemplate) {
v, ok := annotations[preservedRuntimeClassNameAnnotation]
if !ok || len(validation.IsDNS1123Subdomain(v)) > 0 {
return
}
if dst.PodOverrides == nil {
dst.PodOverrides = &v1alpha2.PodOverrides{}
}
if dst.PodOverrides.RuntimeClassName == nil {
dst.PodOverrides.RuntimeClassName = &v
}
}

// setPreservedGitHubWebhookExcludeFilters records
// spec.when.githubWebhook.excludeFilters in an annotation so a v1alpha1 client
// that writes the object back does not drop it.
Expand Down
Loading
Loading