diff --git a/benchmarking/locust/common/ateapi_pb2.py b/benchmarking/locust/common/ateapi_pb2.py index d6f87e1cd4..050f39aaec 100644 --- a/benchmarking/locust/common/ateapi_pb2.py +++ b/benchmarking/locust/common/ateapi_pb2.py @@ -40,7 +40,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x61teapi.proto\x12\x06\x61teapi\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x86\x01\n\x11LocalSnapshotInfo\x12\x15\n\rsnapshot_name\x18\x01 \x01(\t\x12%\n\x1dnode_vms_with_local_snapshots\x18\x02 \x03(\t\x12\x33\n\rcontent_scope\x18\x03 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\"w\n\x08Selector\x12\x37\n\x0cmatch_labels\x18\x01 \x03(\x0b\x32!.ateapi.Selector.MatchLabelsEntry\x1a\x32\n\x10MatchLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb2\x01\n\x10ResourceMetadata\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0b\n\x03uid\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\x03\x12/\n\x0b\x63reate_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12/\n\x0bupdate_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xdc\x02\n\x0e\x45xternalVolume\x12\x13\n\x0bvolume_name\x18\x01 \x01(\t\x12\x19\n\x11storage_volume_id\x18\x02 \x01(\t\x12\x13\n\x0bvolume_type\x18\x03 \x01(\t\x12-\n\x06status\x18\x04 \x01(\x0e\x32\x1d.ateapi.ExternalVolume.Status\x12\x41\n\x0evolume_context\x18\x05 \x03(\x0b\x32).ateapi.ExternalVolume.VolumeContextEntry\x1a\x34\n\x12VolumeContextEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x12\n\x0eSTATUS_PENDING\x10\x01\x12\x12\n\x0eSTATUS_CREATED\x10\x02\x12\x13\n\x0fSTATUS_DELETING\x10\x03\"\xde\x01\n\x05\x41\x63tor\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12)\n\x0e\x61\x63tor_template\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12)\n\x0fworker_selector\x18\x05 \x01(\x0b\x32\x10.ateapi.Selector\x12.\n\x13source_snapshot_tag\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12#\n\x06status\x18\x07 \x01(\x0b\x32\x13.ateapi.ActorStatus\"]\n\x0c\x45gressPolicy\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12!\n\x05rules\x18\x02 \x03(\x0b\x32\x12.ateapi.EgressRule\"\x82\x01\n\nEgressRule\x12\'\n\thostnames\x18\x01 \x01(\x0b\x32\x14.ateapi.HostnameRule\x12&\n\tip_blocks\x18\x02 \x01(\x0b\x32\x13.ateapi.IPBlockRule\x12#\n\x03\x61ll\x18\x03 \x01(\x0b\x32\x16.google.protobuf.Empty\"L\n\x0cHostnameRule\x12\x10\n\x08patterns\x18\x01 \x03(\t\x12*\n\x07\x65\x66\x66\x65\x63ts\x18\x02 \x01(\x0b\x32\x19.ateapi.EgressRuleEffects\"\x1c\n\x0bIPBlockRule\x12\r\n\x05\x63idrs\x18\x01 \x03(\t\"U\n\x11\x45gressRuleEffects\x12@\n\x15inject_static_headers\x18\x01 \x03(\x0b\x32!.ateapi.CredentialHeaderInjection\"S\n\x19\x43redentialHeaderInjection\x12\x0e\n\x06header\x18\x01 \x01(\t\x12\x0e\n\x06prefix\x18\x02 \x01(\t\x12\x16\n\x0e\x63redential_uri\x18\x03 \x01(\t\"\x8a\x04\n\x0b\x41\x63torStatus\x12!\n\x05state\x18\x01 \x01(\x0e\x32\x12.ateapi.ActorState\x12\x33\n\x11worker_assignment\x18\x02 \x01(\x0b\x32\x18.ateapi.WorkerAssignment\x12!\n\x19in_progress_snapshot_name\x18\x03 \x01(\t\x12*\n\x0flatest_snapshot\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x36\n\x13local_snapshot_info\x18\x05 \x01(\x0b\x32\x19.ateapi.LocalSnapshotInfo\x12\x31\n)in_progress_snapshot_source_actor_version\x18\x06 \x01(\x03\x12-\n\ractor_volumes\x18\x07 \x03(\x0b\x32\x16.ateapi.ExternalVolume\x12\'\n\x1fin_progress_local_snapshot_name\x18\x08 \x01(\t\x12:\n\x0fsource_snapshot\x18\t \x01(\x0b\x32!.ateapi.ActorSourceSnapshotStatus\x12\x31\n\x16\x63urrent_actor_template\x18\n \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\"\n\x1a\x63urrent_actor_template_uid\x18\x0b \x01(\t\"V\n\x19\x41\x63torSourceSnapshotStatus\x12#\n\x08snapshot\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x14\n\x0csnapshot_uid\x18\x02 \x01(\t\"\xa7\x01\n\x10WorkerAssignment\x12!\n\x06worker\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x18\n\x10worker_namespace\x18\x01 \x01(\t\x12\x13\n\x0bworker_pool\x18\x02 \x01(\t\x12\x12\n\nworker_pod\x18\x03 \x01(\t\x12\x16\n\x0eworker_pod_uid\x18\x04 \x01(\t\x12\x15\n\rworker_pod_ip\x18\x05 \x01(\t\"h\n\rActorSnapshot\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12+\n\x06status\x18\x02 \x01(\x0b\x32\x1b.ateapi.ActorSnapshotStatus\"\x88\x02\n\x13\x41\x63torSnapshotStatus\x12\'\n\x0csource_actor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x18\n\x10source_actor_uid\x18\x02 \x01(\t\x12\x1c\n\x14source_actor_version\x18\x03 \x01(\x03\x12\x1a\n\x12\x61\x63tor_template_uid\x18\x06 \x01(\t\x12\x33\n\rcontent_scope\x18\x07 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12\x14\n\x0csnapshot_uri\x18\x08 \x01(\t\x12)\n\x0e\x61\x63tor_template\x18\t \x01(\x0b\x32\x11.ateapi.ObjectRef\"\x91\x01\n\x10\x41\x63torSnapshotTag\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12#\n\x08snapshot\x18\x02 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12,\n\x05scope\x18\x03 \x01(\x0e\x32\x1d.ateapi.ActorSnapshotTagScope\"6\n\x08\x41tespace\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\"+\n\tObjectRef\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"\xe3\x02\n\rActorTemplate\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12)\n\x0fworker_selector\x18\x02 \x01(\x0b\x32\x10.ateapi.Selector\x12%\n\ncontainers\x18\x03 \x03(\x0b\x32\x11.ateapi.Container\x12\x1f\n\x07volumes\x18\x04 \x03(\x0b\x32\x0e.ateapi.Volume\x12\x31\n\x10snapshots_config\x18\x05 \x01(\x0b\x32\x17.ateapi.SnapshotsConfig\x12-\n\x0esandbox_config\x18\x06 \x01(\x0b\x32\x15.ateapi.SandboxConfig\x12$\n\tresources\x18\x07 \x01(\x0b\x32\x11.ateapi.Resources\x12+\n\x06status\x18\x08 \x01(\x0b\x32\x1b.ateapi.ActorTemplateStatus\"+\n\tResources\x12\x1e\n\x06limits\x18\x01 \x03(\x0b\x32\x0e.ateapi.Limits\"(\n\x06Limits\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08quantity\x18\x02 \x01(\t\"\x96\x01\n\x14GoldenSnapshotStatus\x12*\n\x0fgolden_snapshot\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12;\n\x17take_golden_snapshot_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x15\n\rerror_message\x18\x03 \x01(\t\"S\n\x13\x41\x63torTemplateStatus\x12<\n\x16golden_snapshot_status\x18\x01 \x01(\x0b\x32\x1c.ateapi.GoldenSnapshotStatus\"Q\n\rSandboxConfig\x12+\n\rsandbox_class\x18\x01 \x01(\x0e\x32\x14.ateapi.SandboxClass\x12\x13\n\x0b\x63onfig_name\x18\x02 \x01(\t\"\xb7\x01\n\x0fSnapshotsConfig\x12.\n\x08on_pause\x18\x01 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12/\n\ton_commit\x18\x02 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12)\n\ton_resume\x18\x03 \x01(\x0b\x32\x16.ateapi.OnResumeConfig\x12\x18\n\x10storage_location\x18\x04 \x01(\t\"9\n\x0eOnResumeConfig\x12\'\n\tfrom_data\x18\x01 \x01(\x0e\x32\x14.ateapi.ResumeSource\"\x92\x02\n\tContainer\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05image\x18\x02 \x01(\t\x12\x0f\n\x07\x63ommand\x18\x03 \x03(\t\x12\x0c\n\x04\x61rgs\x18\x04 \x03(\t\x12\x1b\n\x03\x65nv\x18\x05 \x03(\x0b\x32\x0e.ateapi.EnvVar\x12\'\n\x06readyz\x18\x06 \x01(\x0b\x32\x17.ateapi.ContainerReadyz\x12*\n\rvolume_mounts\x18\x07 \x03(\x0b\x32\x13.ateapi.VolumeMount\x12\x31\n\x10security_context\x18\x08 \x01(\x0b\x32\x17.ateapi.SecurityContext\x12$\n\tresources\x18\t \x01(\x0b\x32\x11.ateapi.Resources\"=\n\x0fSecurityContext\x12*\n\x0c\x63\x61pabilities\x18\x01 \x01(\x0b\x32\x14.ateapi.Capabilities\")\n\x0c\x43\x61pabilities\x12\x0b\n\x03\x61\x64\x64\x18\x01 \x03(\t\x12\x0c\n\x04\x64rop\x18\x02 \x03(\t\"%\n\x06\x45nvVar\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"S\n\x0f\x43ontainerReadyz\x12\'\n\x08http_get\x18\x01 \x01(\x0b\x32\x15.ateapi.HTTPGetAction\x12\x17\n\x0ftimeout_seconds\x18\x02 \x01(\x05\"+\n\rHTTPGetAction\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xec\x01\n\x06Volume\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x33\n\x0b\x64urable_dir\x18\x02 \x01(\x0b\x32\x1e.ateapi.DurableDirVolumeSource\x12@\n\x18\x65xternal_volume_template\x18\x03 \x01(\x0b\x32\x1e.ateapi.ExternalVolumeTemplate\x12\x33\n\x0bsystem_info\x18\x05 \x01(\x0b\x32\x1e.ateapi.SystemInfoVolumeSource\x12(\n\x05image\x18\x06 \x01(\x0b\x32\x19.ateapi.ImageVolumeSource\"&\n\x11ImageVolumeSource\x12\x11\n\treference\x18\x01 \x01(\t\"\x18\n\x16\x44urableDirVolumeSource\"F\n\x16\x45xternalVolumeTemplate\x12\x10\n\x08\x63\x61pacity\x18\x01 \x01(\t\x12\x1a\n\x12storage_class_name\x18\x02 \x01(\t\"L\n\x16SystemInfoVolumeSource\x12\x32\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x1c.ateapi.SystemInfoDataSource\"\x84\x01\n\x14SystemInfoDataSource\x12\x37\n\x0e\x61\x63tor_metadata\x18\x01 \x01(\x0b\x32\x1f.ateapi.ActorMetadataDataSource\x12\x33\n\x0ctrust_bundle\x18\x02 \x01(\x0b\x32\x1d.ateapi.TrustBundleDataSource\"C\n\x17\x41\x63torMetadataDataSource\x12(\n\x05items\x18\x01 \x03(\x0b\x32\x19.ateapi.ActorMetadataItem\"L\n\x11\x41\x63torMetadataItem\x12)\n\x05\x66ield\x18\x01 \x01(\x0e\x32\x1a.ateapi.ActorMetadataField\x12\x0c\n\x04path\x18\x02 \x01(\t\"3\n\x15TrustBundleDataSource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\"/\n\x0bVolumeMount\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nmount_path\x18\x02 \x01(\t\";\n\x15\x43reateAtespaceRequest\x12\"\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x10.ateapi.Atespace\"9\n\x12GetAtespaceRequest\x12#\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"=\n\x14ListAtespacesRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\"U\n\x15ListAtespacesResponse\x12#\n\tatespaces\x18\x01 \x03(\x0b\x32\x10.ateapi.Atespace\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"<\n\x15\x44\x65leteAtespaceRequest\x12#\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"K\n\x1a\x43reateActorTemplateRequest\x12-\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x15.ateapi.ActorTemplate\"D\n\x17GetActorTemplateRequest\x12)\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"T\n\x19ListActorTemplatesRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"e\n\x1aListActorTemplatesResponse\x12.\n\x0f\x61\x63tor_templates\x18\x01 \x03(\x0b\x32\x15.ateapi.ActorTemplate\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"G\n\x1a\x44\x65leteActorTemplateRequest\x12)\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"3\n\x0fGetActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"2\n\x12\x43reateActorRequest\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"2\n\x12UpdateActorRequest\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"7\n\x13SuspendActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"4\n\x14SuspendActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"5\n\x11PauseActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"2\n\x12PauseActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"D\n\x12ResumeActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x0c\n\x04\x62oot\x18\x02 \x01(\x08\"D\n\x13ResumeActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\x12\x0f\n\x07resumed\x18\x02 \x01(\x08\"I\n\x12\x44\x65leteActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tany_state\x18\x02 \x01(\x08\"?\n\x1bGetActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"o\n\x1e\x43reateActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12+\n\regress_policy\x18\x02 \x01(\x0b\x32\x14.ateapi.EgressPolicy\"o\n\x1eUpdateActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12+\n\regress_policy\x18\x02 \x01(\x0b\x32\x14.ateapi.EgressPolicy\"B\n\x1e\x44\x65leteActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"D\n\x17GetActorSnapshotRequest\x12)\n\x0e\x61\x63tor_snapshot\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"K\n\x1aGetActorSnapshotTagRequest\x12-\n\x12\x61\x63tor_snapshot_tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"T\n\x19ListActorSnapshotsRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"e\n\x1aListActorSnapshotsResponse\x12.\n\x0f\x61\x63tor_snapshots\x18\x01 \x03(\x0b\x32\x15.ateapi.ActorSnapshot\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"U\n\x1d\x43reateActorSnapshotTagRequest\x12\x34\n\x12\x61\x63tor_snapshot_tag\x18\x01 \x01(\x0b\x32\x18.ateapi.ActorSnapshotTag\"U\n\x1dUpdateActorSnapshotTagRequest\x12\x34\n\x12\x61\x63tor_snapshot_tag\x18\x01 \x01(\x0b\x32\x18.ateapi.ActorSnapshotTag\"N\n\x1d\x44\x65leteActorSnapshotTagRequest\x12-\n\x12\x61\x63tor_snapshot_tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"-\n\rDeleteOptions\x12\x0f\n\x07version\x18\x01 \x01(\x03\x12\x0b\n\x03uid\x18\x02 \x01(\t\";\n\x12ListWorkersRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\"O\n\x13ListWorkersResponse\x12\x1f\n\x07workers\x18\x01 \x03(\x0b\x32\x0e.ateapi.Worker\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"5\n\x10GetWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"5\n\x13\x43reateWorkerRequest\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"5\n\x13UpdateWorkerRequest\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"`\n\x13\x44\x65leteWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12&\n\x07options\x18\x02 \x01(\x0b\x32\x15.ateapi.DeleteOptions\"7\n\x12\x44rainWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"L\n\x11ListActorsRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"L\n\x12ListActorsResponse\x12\x1d\n\x06\x61\x63tors\x18\x01 \x03(\x0b\x32\r.ateapi.Actor\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"\xf0\x02\n\x06Worker\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12\x18\n\x10worker_namespace\x18\x02 \x01(\t\x12\x13\n\x0bworker_pool\x18\x03 \x01(\t\x12\x12\n\nworker_pod\x18\x04 \x01(\t\x12\x16\n\x0eworker_pod_uid\x18\x05 \x01(\t\x12\x11\n\tnode_name\x18\x06 \x01(\t\x12\n\n\x02ip\x18\x07 \x01(\t\x12\x15\n\rsandbox_class\x18\x08 \x01(\t\x12*\n\x06labels\x18\t \x03(\x0b\x32\x1a.ateapi.Worker.LabelsEntry\x12(\n\x08\x63\x61pacity\x18\n \x01(\x0b\x32\x16.ateapi.WorkerCapacity\x12$\n\x06status\x18\x0b \x01(\x0b\x32\x14.ateapi.WorkerStatus\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"_\n\x0cWorkerStatus\x12\"\n\x05state\x18\x01 \x01(\x0e\x32\x13.ateapi.WorkerState\x12+\n\nassignment\x18\x02 \x01(\x0b\x32\x17.ateapi.ActorAssignment\"9\n\x0eWorkerCapacity\x12\x11\n\tcpu_milli\x18\x01 \x01(\x03\x12\x14\n\x0cmemory_bytes\x18\x02 \x01(\x03\"u\n\x0f\x41\x63torAssignment\x12 \n\x05\x61\x63tor\x18\x02 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tactor_uid\x18\x03 \x01(\t\x12-\n\x12\x61\x63tor_template_ref\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\"[\n\x0eMintJWTRequest\x12\x10\n\x08\x61udience\x18\x01 \x03(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12\x12\n\nactor_name\x18\x03 \x01(\t\x12\x11\n\tactor_uid\x18\x04 \x01(\t\"$\n\x0fMintJWTResponse\x12\x11\n\tactor_jwt\x18\x01 \x01(\t\"\xa7\x01\n\x0fMintCertRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12#\n\x1b\x63\x65rtificate_signing_request\x18\x02 \x01(\x0c\x12\x1a\n\x12\x65xpected_actor_uid\x18\x03 \x01(\t\x12\x30\n\x07purpose\x18\x04 \x01(\x0e\x32\x1f.ateapi.ActorCertificatePurpose\".\n\x10MintCertResponse\x12\x1a\n\x12\x61\x63tor_certificates\x18\x01 \x03(\x0c*\x80\x01\n\x14SnapshotContentScope\x12&\n\"SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bSNAPSHOT_CONTENT_SCOPE_FULL\x10\x01\x12\x1f\n\x1bSNAPSHOT_CONTENT_SCOPE_DATA\x10\x02*\x90\x01\n\x15\x41\x63torSnapshotTagScope\x12(\n$ACTOR_SNAPSHOT_TAG_SCOPE_UNSPECIFIED\x10\x00\x12%\n!ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE\x10\x01\x12&\n\"ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED\x10\x02*\xf7\x01\n\nActorState\x12\x1b\n\x17\x41\x43TOR_STATE_UNSPECIFIED\x10\x00\x12\x18\n\x14\x41\x43TOR_STATE_RESUMING\x10\x01\x12\x17\n\x13\x41\x43TOR_STATE_RUNNING\x10\x02\x12\x1a\n\x16\x41\x43TOR_STATE_SUSPENDING\x10\x03\x12\x19\n\x15\x41\x43TOR_STATE_SUSPENDED\x10\x04\x12\x17\n\x13\x41\x43TOR_STATE_PAUSING\x10\x05\x12\x16\n\x12\x41\x43TOR_STATE_PAUSED\x10\x06\x12\x17\n\x13\x41\x43TOR_STATE_CRASHED\x10\x07\x12\x18\n\x14\x41\x43TOR_STATE_DELETING\x10\x08*b\n\x0cSandboxClass\x12\x1d\n\x19SANDBOX_CLASS_UNSPECIFIED\x10\x00\x12\x18\n\x14SANDBOX_CLASS_GVISOR\x10\x01\x12\x19\n\x15SANDBOX_CLASS_MICROVM\x10\x02*d\n\x0cResumeSource\x12\x1d\n\x19RESUME_SOURCE_UNSPECIFIED\x10\x00\x12\x1b\n\x17RESUME_SOURCE_COLD_BOOT\x10\x01\x12\x18\n\x14RESUME_SOURCE_GOLDEN\x10\x02*\x9a\x01\n\x12\x41\x63torMetadataField\x12$\n ACTOR_METADATA_FIELD_UNSPECIFIED\x10\x00\x12\x1d\n\x19\x41\x43TOR_METADATA_FIELD_NAME\x10\x01\x12!\n\x1d\x41\x43TOR_METADATA_FIELD_ATESPACE\x10\x02\x12\x1c\n\x18\x41\x43TOR_METADATA_FIELD_UID\x10\x03*_\n\x0bWorkerState\x12\x1c\n\x18WORKER_STATE_UNSPECIFIED\x10\x00\x12\x17\n\x13WORKER_STATE_ACTIVE\x10\x01\x12\x19\n\x15WORKER_STATE_DRAINING\x10\x02*k\n\x17\x41\x63torCertificatePurpose\x12)\n%ACTOR_CERTIFICATE_PURPOSE_UNSPECIFIED\x10\x00\x12%\n!ACTOR_CERTIFICATE_PURPOSE_ATUNNEL\x10\x01\x32\x84\x13\n\x07\x43ontrol\x12\x34\n\x08GetActor\x12\x17.ateapi.GetActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n\x0b\x43reateActor\x12\x1a.ateapi.CreateActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n\x0bUpdateActor\x12\x1a.ateapi.UpdateActorRequest\x1a\r.ateapi.Actor\"\x00\x12K\n\x0cSuspendActor\x12\x1b.ateapi.SuspendActorRequest\x1a\x1c.ateapi.SuspendActorResponse\"\x00\x12\x45\n\nPauseActor\x12\x19.ateapi.PauseActorRequest\x1a\x1a.ateapi.PauseActorResponse\"\x00\x12H\n\x0bResumeActor\x12\x1a.ateapi.ResumeActorRequest\x1a\x1b.ateapi.ResumeActorResponse\"\x00\x12:\n\x0b\x44\x65leteActor\x12\x1a.ateapi.DeleteActorRequest\x1a\r.ateapi.Actor\"\x00\x12S\n\x14GetActorEgressPolicy\x12#.ateapi.GetActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17\x43reateActorEgressPolicy\x12&.ateapi.CreateActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17UpdateActorEgressPolicy\x12&.ateapi.UpdateActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17\x44\x65leteActorEgressPolicy\x12&.ateapi.DeleteActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12L\n\x10GetActorSnapshot\x12\x1f.ateapi.GetActorSnapshotRequest\x1a\x15.ateapi.ActorSnapshot\"\x00\x12U\n\x13GetActorSnapshotTag\x12\".ateapi.GetActorSnapshotTagRequest\x1a\x18.ateapi.ActorSnapshotTag\"\x00\x12]\n\x12ListActorSnapshots\x12!.ateapi.ListActorSnapshotsRequest\x1a\".ateapi.ListActorSnapshotsResponse\"\x00\x12[\n\x16\x43reateActorSnapshotTag\x12%.ateapi.CreateActorSnapshotTagRequest\x1a\x18.ateapi.ActorSnapshotTag\"\x00\x12[\n\x16UpdateActorSnapshotTag\x12%.ateapi.UpdateActorSnapshotTagRequest\x1a\x18.ateapi.ActorSnapshotTag\"\x00\x12[\n\x16\x44\x65leteActorSnapshotTag\x12%.ateapi.DeleteActorSnapshotTagRequest\x1a\x18.ateapi.ActorSnapshotTag\"\x00\x12H\n\x0bListWorkers\x12\x1a.ateapi.ListWorkersRequest\x1a\x1b.ateapi.ListWorkersResponse\"\x00\x12\x37\n\tGetWorker\x12\x18.ateapi.GetWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0c\x43reateWorker\x12\x1b.ateapi.CreateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0cUpdateWorker\x12\x1b.ateapi.UpdateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0c\x44\x65leteWorker\x12\x1b.ateapi.DeleteWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12;\n\x0b\x44rainWorker\x12\x1a.ateapi.DrainWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12\x45\n\nListActors\x12\x19.ateapi.ListActorsRequest\x1a\x1a.ateapi.ListActorsResponse\"\x00\x12\x43\n\x0e\x43reateAtespace\x12\x1d.ateapi.CreateAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12=\n\x0bGetAtespace\x12\x1a.ateapi.GetAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12N\n\rListAtespaces\x12\x1c.ateapi.ListAtespacesRequest\x1a\x1d.ateapi.ListAtespacesResponse\"\x00\x12\x43\n\x0e\x44\x65leteAtespace\x12\x1d.ateapi.DeleteAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12R\n\x13\x43reateActorTemplate\x12\".ateapi.CreateActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x12L\n\x10GetActorTemplate\x12\x1f.ateapi.GetActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x12]\n\x12ListActorTemplates\x12!.ateapi.ListActorTemplatesRequest\x1a\".ateapi.ListActorTemplatesResponse\"\x00\x12R\n\x13\x44\x65leteActorTemplate\x12\".ateapi.DeleteActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x32\x8a\x01\n\rActorIdentity\x12:\n\x07MintJWT\x12\x16.ateapi.MintJWTRequest\x1a\x17.ateapi.MintJWTResponse\x12=\n\x08MintCert\x12\x17.ateapi.MintCertRequest\x1a\x18.ateapi.MintCertResponseB9Z7github.com/agent-substrate/substrate/pkg/proto/ateapipbb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x61teapi.proto\x12\x06\x61teapi\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x86\x01\n\x11LocalSnapshotInfo\x12\x15\n\rsnapshot_name\x18\x01 \x01(\t\x12%\n\x1dnode_vms_with_local_snapshots\x18\x02 \x03(\t\x12\x33\n\rcontent_scope\x18\x03 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\"w\n\x08Selector\x12\x37\n\x0cmatch_labels\x18\x01 \x03(\x0b\x32!.ateapi.Selector.MatchLabelsEntry\x1a\x32\n\x10MatchLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb2\x01\n\x10ResourceMetadata\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0b\n\x03uid\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\x03\x12/\n\x0b\x63reate_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12/\n\x0bupdate_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xdc\x02\n\x0e\x45xternalVolume\x12\x13\n\x0bvolume_name\x18\x01 \x01(\t\x12\x19\n\x11storage_volume_id\x18\x02 \x01(\t\x12\x13\n\x0bvolume_type\x18\x03 \x01(\t\x12-\n\x06status\x18\x04 \x01(\x0e\x32\x1d.ateapi.ExternalVolume.Status\x12\x41\n\x0evolume_context\x18\x05 \x03(\x0b\x32).ateapi.ExternalVolume.VolumeContextEntry\x1a\x34\n\x12VolumeContextEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x12\n\x0eSTATUS_PENDING\x10\x01\x12\x12\n\x0eSTATUS_CREATED\x10\x02\x12\x13\n\x0fSTATUS_DELETING\x10\x03\"\xde\x01\n\x05\x41\x63tor\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12)\n\x0e\x61\x63tor_template\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12)\n\x0fworker_selector\x18\x05 \x01(\x0b\x32\x10.ateapi.Selector\x12.\n\x13source_snapshot_tag\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12#\n\x06status\x18\x07 \x01(\x0b\x32\x13.ateapi.ActorStatus\"]\n\x0c\x45gressPolicy\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12!\n\x05rules\x18\x02 \x03(\x0b\x32\x12.ateapi.EgressRule\"\x82\x01\n\nEgressRule\x12\'\n\thostnames\x18\x01 \x01(\x0b\x32\x14.ateapi.HostnameRule\x12&\n\tip_blocks\x18\x02 \x01(\x0b\x32\x13.ateapi.IPBlockRule\x12#\n\x03\x61ll\x18\x03 \x01(\x0b\x32\x16.google.protobuf.Empty\"L\n\x0cHostnameRule\x12\x10\n\x08patterns\x18\x01 \x03(\t\x12*\n\x07\x65\x66\x66\x65\x63ts\x18\x02 \x01(\x0b\x32\x19.ateapi.EgressRuleEffects\"\x1c\n\x0bIPBlockRule\x12\r\n\x05\x63idrs\x18\x01 \x03(\t\"U\n\x11\x45gressRuleEffects\x12@\n\x15inject_static_headers\x18\x01 \x03(\x0b\x32!.ateapi.CredentialHeaderInjection\"S\n\x19\x43redentialHeaderInjection\x12\x0e\n\x06header\x18\x01 \x01(\t\x12\x0e\n\x06prefix\x18\x02 \x01(\t\x12\x16\n\x0e\x63redential_uri\x18\x03 \x01(\t\"\x8a\x04\n\x0b\x41\x63torStatus\x12!\n\x05state\x18\x01 \x01(\x0e\x32\x12.ateapi.ActorState\x12\x33\n\x11worker_assignment\x18\x02 \x01(\x0b\x32\x18.ateapi.WorkerAssignment\x12!\n\x19in_progress_snapshot_name\x18\x03 \x01(\t\x12*\n\x0flatest_snapshot\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x36\n\x13local_snapshot_info\x18\x05 \x01(\x0b\x32\x19.ateapi.LocalSnapshotInfo\x12\x31\n)in_progress_snapshot_source_actor_version\x18\x06 \x01(\x03\x12-\n\ractor_volumes\x18\x07 \x03(\x0b\x32\x16.ateapi.ExternalVolume\x12\'\n\x1fin_progress_local_snapshot_name\x18\x08 \x01(\t\x12:\n\x0fsource_snapshot\x18\t \x01(\x0b\x32!.ateapi.ActorSourceSnapshotStatus\x12\x31\n\x16\x63urrent_actor_template\x18\n \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\"\n\x1a\x63urrent_actor_template_uid\x18\x0b \x01(\t\"V\n\x19\x41\x63torSourceSnapshotStatus\x12#\n\x08snapshot\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x14\n\x0csnapshot_uid\x18\x02 \x01(\t\"\xa7\x01\n\x10WorkerAssignment\x12!\n\x06worker\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x18\n\x10worker_namespace\x18\x01 \x01(\t\x12\x13\n\x0bworker_pool\x18\x02 \x01(\t\x12\x12\n\nworker_pod\x18\x03 \x01(\t\x12\x16\n\x0eworker_pod_uid\x18\x04 \x01(\t\x12\x15\n\rworker_pod_ip\x18\x05 \x01(\t\"h\n\rActorSnapshot\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12+\n\x06status\x18\x02 \x01(\x0b\x32\x1b.ateapi.ActorSnapshotStatus\"\x88\x02\n\x13\x41\x63torSnapshotStatus\x12\'\n\x0csource_actor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x18\n\x10source_actor_uid\x18\x02 \x01(\t\x12\x1c\n\x14source_actor_version\x18\x03 \x01(\x03\x12\x1a\n\x12\x61\x63tor_template_uid\x18\x06 \x01(\t\x12\x33\n\rcontent_scope\x18\x07 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12\x14\n\x0csnapshot_uri\x18\x08 \x01(\t\x12)\n\x0e\x61\x63tor_template\x18\t \x01(\x0b\x32\x11.ateapi.ObjectRef\"\x91\x01\n\x10\x41\x63torSnapshotTag\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12#\n\x08snapshot\x18\x02 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12,\n\x05scope\x18\x03 \x01(\x0e\x32\x1d.ateapi.ActorSnapshotTagScope\"6\n\x08\x41tespace\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\"+\n\tObjectRef\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"\xe3\x02\n\rActorTemplate\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12)\n\x0fworker_selector\x18\x02 \x01(\x0b\x32\x10.ateapi.Selector\x12%\n\ncontainers\x18\x03 \x03(\x0b\x32\x11.ateapi.Container\x12\x1f\n\x07volumes\x18\x04 \x03(\x0b\x32\x0e.ateapi.Volume\x12\x31\n\x10snapshots_config\x18\x05 \x01(\x0b\x32\x17.ateapi.SnapshotsConfig\x12-\n\x0esandbox_config\x18\x06 \x01(\x0b\x32\x15.ateapi.SandboxConfig\x12$\n\tresources\x18\x07 \x01(\x0b\x32\x11.ateapi.Resources\x12+\n\x06status\x18\x08 \x01(\x0b\x32\x1b.ateapi.ActorTemplateStatus\"+\n\tResources\x12\x1e\n\x06limits\x18\x01 \x03(\x0b\x32\x0e.ateapi.Limits\"(\n\x06Limits\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08quantity\x18\x02 \x01(\t\"\x96\x01\n\x14GoldenSnapshotStatus\x12*\n\x0fgolden_snapshot\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12;\n\x17take_golden_snapshot_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x15\n\rerror_message\x18\x03 \x01(\t\"S\n\x13\x41\x63torTemplateStatus\x12<\n\x16golden_snapshot_status\x18\x01 \x01(\x0b\x32\x1c.ateapi.GoldenSnapshotStatus\"Q\n\rSandboxConfig\x12+\n\rsandbox_class\x18\x01 \x01(\x0e\x32\x14.ateapi.SandboxClass\x12\x13\n\x0b\x63onfig_name\x18\x02 \x01(\t\"\xb7\x01\n\x0fSnapshotsConfig\x12.\n\x08on_pause\x18\x01 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12/\n\ton_commit\x18\x02 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12)\n\ton_resume\x18\x03 \x01(\x0b\x32\x16.ateapi.OnResumeConfig\x12\x18\n\x10storage_location\x18\x04 \x01(\t\"9\n\x0eOnResumeConfig\x12\'\n\tfrom_data\x18\x01 \x01(\x0e\x32\x14.ateapi.ResumeSource\"\x92\x02\n\tContainer\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05image\x18\x02 \x01(\t\x12\x0f\n\x07\x63ommand\x18\x03 \x03(\t\x12\x0c\n\x04\x61rgs\x18\x04 \x03(\t\x12\x1b\n\x03\x65nv\x18\x05 \x03(\x0b\x32\x0e.ateapi.EnvVar\x12\'\n\x06readyz\x18\x06 \x01(\x0b\x32\x17.ateapi.ContainerReadyz\x12*\n\rvolume_mounts\x18\x07 \x03(\x0b\x32\x13.ateapi.VolumeMount\x12\x31\n\x10security_context\x18\x08 \x01(\x0b\x32\x17.ateapi.SecurityContext\x12$\n\tresources\x18\t \x01(\x0b\x32\x11.ateapi.Resources\"=\n\x0fSecurityContext\x12*\n\x0c\x63\x61pabilities\x18\x01 \x01(\x0b\x32\x14.ateapi.Capabilities\")\n\x0c\x43\x61pabilities\x12\x0b\n\x03\x61\x64\x64\x18\x01 \x03(\t\x12\x0c\n\x04\x64rop\x18\x02 \x03(\t\"%\n\x06\x45nvVar\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"S\n\x0f\x43ontainerReadyz\x12\'\n\x08http_get\x18\x01 \x01(\x0b\x32\x15.ateapi.HTTPGetAction\x12\x17\n\x0ftimeout_seconds\x18\x02 \x01(\x05\"+\n\rHTTPGetAction\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xec\x01\n\x06Volume\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x33\n\x0b\x64urable_dir\x18\x02 \x01(\x0b\x32\x1e.ateapi.DurableDirVolumeSource\x12@\n\x18\x65xternal_volume_template\x18\x03 \x01(\x0b\x32\x1e.ateapi.ExternalVolumeTemplate\x12\x33\n\x0bsystem_info\x18\x05 \x01(\x0b\x32\x1e.ateapi.SystemInfoVolumeSource\x12(\n\x05image\x18\x06 \x01(\x0b\x32\x19.ateapi.ImageVolumeSource\"&\n\x11ImageVolumeSource\x12\x11\n\treference\x18\x01 \x01(\t\"\x18\n\x16\x44urableDirVolumeSource\"F\n\x16\x45xternalVolumeTemplate\x12\x10\n\x08\x63\x61pacity\x18\x01 \x01(\t\x12\x1a\n\x12storage_class_name\x18\x02 \x01(\t\"L\n\x16SystemInfoVolumeSource\x12\x32\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x1c.ateapi.SystemInfoDataSource\"\x84\x01\n\x14SystemInfoDataSource\x12\x37\n\x0e\x61\x63tor_metadata\x18\x01 \x01(\x0b\x32\x1f.ateapi.ActorMetadataDataSource\x12\x33\n\x0ctrust_bundle\x18\x02 \x01(\x0b\x32\x1d.ateapi.TrustBundleDataSource\"C\n\x17\x41\x63torMetadataDataSource\x12(\n\x05items\x18\x01 \x03(\x0b\x32\x19.ateapi.ActorMetadataItem\"L\n\x11\x41\x63torMetadataItem\x12)\n\x05\x66ield\x18\x01 \x01(\x0e\x32\x1a.ateapi.ActorMetadataField\x12\x0c\n\x04path\x18\x02 \x01(\t\"3\n\x15TrustBundleDataSource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\"/\n\x0bVolumeMount\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nmount_path\x18\x02 \x01(\t\";\n\x15\x43reateAtespaceRequest\x12\"\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x10.ateapi.Atespace\"9\n\x12GetAtespaceRequest\x12#\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"=\n\x14ListAtespacesRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\"U\n\x15ListAtespacesResponse\x12#\n\tatespaces\x18\x01 \x03(\x0b\x32\x10.ateapi.Atespace\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"<\n\x15\x44\x65leteAtespaceRequest\x12#\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"K\n\x1a\x43reateActorTemplateRequest\x12-\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x15.ateapi.ActorTemplate\"D\n\x17GetActorTemplateRequest\x12)\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"T\n\x19ListActorTemplatesRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"e\n\x1aListActorTemplatesResponse\x12.\n\x0f\x61\x63tor_templates\x18\x01 \x03(\x0b\x32\x15.ateapi.ActorTemplate\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"G\n\x1a\x44\x65leteActorTemplateRequest\x12)\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"3\n\x0fGetActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"2\n\x12\x43reateActorRequest\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"2\n\x12UpdateActorRequest\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"7\n\x13SuspendActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"4\n\x14SuspendActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"5\n\x11PauseActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"2\n\x12PauseActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"D\n\x12ResumeActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x0c\n\x04\x62oot\x18\x02 \x01(\x08\"D\n\x13ResumeActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\x12\x0f\n\x07resumed\x18\x02 \x01(\x08\"I\n\x12\x44\x65leteActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tany_state\x18\x02 \x01(\x08\"?\n\x1bGetActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"o\n\x1e\x43reateActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12+\n\regress_policy\x18\x02 \x01(\x0b\x32\x14.ateapi.EgressPolicy\"o\n\x1eUpdateActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12+\n\regress_policy\x18\x02 \x01(\x0b\x32\x14.ateapi.EgressPolicy\"B\n\x1e\x44\x65leteActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"D\n\x17GetActorSnapshotRequest\x12)\n\x0e\x61\x63tor_snapshot\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"K\n\x1aGetActorSnapshotTagRequest\x12-\n\x12\x61\x63tor_snapshot_tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"T\n\x19ListActorSnapshotsRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"e\n\x1aListActorSnapshotsResponse\x12.\n\x0f\x61\x63tor_snapshots\x18\x01 \x03(\x0b\x32\x15.ateapi.ActorSnapshot\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"U\n\x1d\x43reateActorSnapshotTagRequest\x12\x34\n\x12\x61\x63tor_snapshot_tag\x18\x01 \x01(\x0b\x32\x18.ateapi.ActorSnapshotTag\"U\n\x1dUpdateActorSnapshotTagRequest\x12\x34\n\x12\x61\x63tor_snapshot_tag\x18\x01 \x01(\x0b\x32\x18.ateapi.ActorSnapshotTag\"N\n\x1d\x44\x65leteActorSnapshotTagRequest\x12-\n\x12\x61\x63tor_snapshot_tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"-\n\rDeleteOptions\x12\x0f\n\x07version\x18\x01 \x01(\x03\x12\x0b\n\x03uid\x18\x02 \x01(\t\"m\n!ListWorkerActorAssignmentsRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"q\n\"ListWorkerActorAssignmentsResponse\x12\x32\n\x11\x61\x63tor_assignments\x18\x01 \x03(\x0b\x32\x17.ateapi.ActorAssignment\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\";\n\x12ListWorkersRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\"O\n\x13ListWorkersResponse\x12\x1f\n\x07workers\x18\x01 \x03(\x0b\x32\x0e.ateapi.Worker\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"5\n\x10GetWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"5\n\x13\x43reateWorkerRequest\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"5\n\x13UpdateWorkerRequest\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"`\n\x13\x44\x65leteWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12&\n\x07options\x18\x02 \x01(\x0b\x32\x15.ateapi.DeleteOptions\"7\n\x12\x44rainWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"L\n\x11ListActorsRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"L\n\x12ListActorsResponse\x12\x1d\n\x06\x61\x63tors\x18\x01 \x03(\x0b\x32\r.ateapi.Actor\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"\xc6\x02\n\x06Worker\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12\x18\n\x10worker_namespace\x18\x02 \x01(\t\x12\x13\n\x0bworker_pool\x18\x03 \x01(\t\x12\x12\n\nworker_pod\x18\x04 \x01(\t\x12\x16\n\x0eworker_pod_uid\x18\x05 \x01(\t\x12\x11\n\tnode_name\x18\x06 \x01(\t\x12\n\n\x02ip\x18\x07 \x01(\t\x12\x15\n\rsandbox_class\x18\x08 \x01(\t\x12*\n\x06labels\x18\t \x03(\x0b\x32\x1a.ateapi.Worker.LabelsEntry\x12$\n\x06status\x18\x0b \x01(\x0b\x32\x14.ateapi.WorkerStatus\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"`\n\x0cWorkerStatus\x12\"\n\x05state\x18\x01 \x01(\x0e\x32\x13.ateapi.WorkerState\x12,\n\nallocation\x18\x02 \x01(\x0b\x32\x18.ateapi.WorkerAllocation\"i\n\x10WorkerAllocation\x12)\n\x08\x63\x61pacity\x18\x01 \x01(\x0b\x32\x17.ateapi.WorkerResources\x12*\n\tallocated\x18\x02 \x01(\x0b\x32\x17.ateapi.WorkerResources\"G\n\x0fWorkerResources\x12$\n\tresources\x18\x01 \x01(\x0b\x32\x11.ateapi.Resources\x12\x0e\n\x06\x61\x63tors\x18\x02 \x01(\x05\"\xc7\x01\n\x0f\x41\x63torAssignment\x12*\n\x08metadata\x18\x06 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12 \n\x05\x61\x63tor\x18\x02 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tactor_uid\x18\x03 \x01(\t\x12-\n\x12\x61\x63tor_template_ref\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12$\n\tresources\x18\x05 \x01(\x0b\x32\x11.ateapi.Resources\"h\n\x18SetWorkerCapacityRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12)\n\x08\x63\x61pacity\x18\x02 \x01(\x0b\x32\x17.ateapi.WorkerResources\";\n\x19SetWorkerCapacityResponse\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"[\n\x0eMintJWTRequest\x12\x10\n\x08\x61udience\x18\x01 \x03(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12\x12\n\nactor_name\x18\x03 \x01(\t\x12\x11\n\tactor_uid\x18\x04 \x01(\t\"$\n\x0fMintJWTResponse\x12\x11\n\tactor_jwt\x18\x01 \x01(\t\"\xa7\x01\n\x0fMintCertRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12#\n\x1b\x63\x65rtificate_signing_request\x18\x02 \x01(\x0c\x12\x1a\n\x12\x65xpected_actor_uid\x18\x03 \x01(\t\x12\x30\n\x07purpose\x18\x04 \x01(\x0e\x32\x1f.ateapi.ActorCertificatePurpose\".\n\x10MintCertResponse\x12\x1a\n\x12\x61\x63tor_certificates\x18\x01 \x03(\x0c*\x80\x01\n\x14SnapshotContentScope\x12&\n\"SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bSNAPSHOT_CONTENT_SCOPE_FULL\x10\x01\x12\x1f\n\x1bSNAPSHOT_CONTENT_SCOPE_DATA\x10\x02*\x90\x01\n\x15\x41\x63torSnapshotTagScope\x12(\n$ACTOR_SNAPSHOT_TAG_SCOPE_UNSPECIFIED\x10\x00\x12%\n!ACTOR_SNAPSHOT_TAG_SCOPE_ATESPACE\x10\x01\x12&\n\"ACTOR_SNAPSHOT_TAG_SCOPE_PUBLISHED\x10\x02*\xf7\x01\n\nActorState\x12\x1b\n\x17\x41\x43TOR_STATE_UNSPECIFIED\x10\x00\x12\x18\n\x14\x41\x43TOR_STATE_RESUMING\x10\x01\x12\x17\n\x13\x41\x43TOR_STATE_RUNNING\x10\x02\x12\x1a\n\x16\x41\x43TOR_STATE_SUSPENDING\x10\x03\x12\x19\n\x15\x41\x43TOR_STATE_SUSPENDED\x10\x04\x12\x17\n\x13\x41\x43TOR_STATE_PAUSING\x10\x05\x12\x16\n\x12\x41\x43TOR_STATE_PAUSED\x10\x06\x12\x17\n\x13\x41\x43TOR_STATE_CRASHED\x10\x07\x12\x18\n\x14\x41\x43TOR_STATE_DELETING\x10\x08*b\n\x0cSandboxClass\x12\x1d\n\x19SANDBOX_CLASS_UNSPECIFIED\x10\x00\x12\x18\n\x14SANDBOX_CLASS_GVISOR\x10\x01\x12\x19\n\x15SANDBOX_CLASS_MICROVM\x10\x02*d\n\x0cResumeSource\x12\x1d\n\x19RESUME_SOURCE_UNSPECIFIED\x10\x00\x12\x1b\n\x17RESUME_SOURCE_COLD_BOOT\x10\x01\x12\x18\n\x14RESUME_SOURCE_GOLDEN\x10\x02*\x9a\x01\n\x12\x41\x63torMetadataField\x12$\n ACTOR_METADATA_FIELD_UNSPECIFIED\x10\x00\x12\x1d\n\x19\x41\x43TOR_METADATA_FIELD_NAME\x10\x01\x12!\n\x1d\x41\x43TOR_METADATA_FIELD_ATESPACE\x10\x02\x12\x1c\n\x18\x41\x43TOR_METADATA_FIELD_UID\x10\x03*_\n\x0bWorkerState\x12\x1c\n\x18WORKER_STATE_UNSPECIFIED\x10\x00\x12\x17\n\x13WORKER_STATE_ACTIVE\x10\x01\x12\x19\n\x15WORKER_STATE_DRAINING\x10\x02*k\n\x17\x41\x63torCertificatePurpose\x12)\n%ACTOR_CERTIFICATE_PURPOSE_UNSPECIFIED\x10\x00\x12%\n!ACTOR_CERTIFICATE_PURPOSE_ATUNNEL\x10\x01\x32\xfb\x13\n\x07\x43ontrol\x12\x34\n\x08GetActor\x12\x17.ateapi.GetActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n\x0b\x43reateActor\x12\x1a.ateapi.CreateActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n\x0bUpdateActor\x12\x1a.ateapi.UpdateActorRequest\x1a\r.ateapi.Actor\"\x00\x12K\n\x0cSuspendActor\x12\x1b.ateapi.SuspendActorRequest\x1a\x1c.ateapi.SuspendActorResponse\"\x00\x12\x45\n\nPauseActor\x12\x19.ateapi.PauseActorRequest\x1a\x1a.ateapi.PauseActorResponse\"\x00\x12H\n\x0bResumeActor\x12\x1a.ateapi.ResumeActorRequest\x1a\x1b.ateapi.ResumeActorResponse\"\x00\x12:\n\x0b\x44\x65leteActor\x12\x1a.ateapi.DeleteActorRequest\x1a\r.ateapi.Actor\"\x00\x12S\n\x14GetActorEgressPolicy\x12#.ateapi.GetActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17\x43reateActorEgressPolicy\x12&.ateapi.CreateActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17UpdateActorEgressPolicy\x12&.ateapi.UpdateActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17\x44\x65leteActorEgressPolicy\x12&.ateapi.DeleteActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12L\n\x10GetActorSnapshot\x12\x1f.ateapi.GetActorSnapshotRequest\x1a\x15.ateapi.ActorSnapshot\"\x00\x12U\n\x13GetActorSnapshotTag\x12\".ateapi.GetActorSnapshotTagRequest\x1a\x18.ateapi.ActorSnapshotTag\"\x00\x12]\n\x12ListActorSnapshots\x12!.ateapi.ListActorSnapshotsRequest\x1a\".ateapi.ListActorSnapshotsResponse\"\x00\x12[\n\x16\x43reateActorSnapshotTag\x12%.ateapi.CreateActorSnapshotTagRequest\x1a\x18.ateapi.ActorSnapshotTag\"\x00\x12[\n\x16UpdateActorSnapshotTag\x12%.ateapi.UpdateActorSnapshotTagRequest\x1a\x18.ateapi.ActorSnapshotTag\"\x00\x12[\n\x16\x44\x65leteActorSnapshotTag\x12%.ateapi.DeleteActorSnapshotTagRequest\x1a\x18.ateapi.ActorSnapshotTag\"\x00\x12H\n\x0bListWorkers\x12\x1a.ateapi.ListWorkersRequest\x1a\x1b.ateapi.ListWorkersResponse\"\x00\x12\x37\n\tGetWorker\x12\x18.ateapi.GetWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0c\x43reateWorker\x12\x1b.ateapi.CreateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0cUpdateWorker\x12\x1b.ateapi.UpdateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0c\x44\x65leteWorker\x12\x1b.ateapi.DeleteWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12;\n\x0b\x44rainWorker\x12\x1a.ateapi.DrainWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12u\n\x1aListWorkerActorAssignments\x12).ateapi.ListWorkerActorAssignmentsRequest\x1a*.ateapi.ListWorkerActorAssignmentsResponse\"\x00\x12\x45\n\nListActors\x12\x19.ateapi.ListActorsRequest\x1a\x1a.ateapi.ListActorsResponse\"\x00\x12\x43\n\x0e\x43reateAtespace\x12\x1d.ateapi.CreateAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12=\n\x0bGetAtespace\x12\x1a.ateapi.GetAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12N\n\rListAtespaces\x12\x1c.ateapi.ListAtespacesRequest\x1a\x1d.ateapi.ListAtespacesResponse\"\x00\x12\x43\n\x0e\x44\x65leteAtespace\x12\x1d.ateapi.DeleteAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12R\n\x13\x43reateActorTemplate\x12\".ateapi.CreateActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x12L\n\x10GetActorTemplate\x12\x1f.ateapi.GetActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x12]\n\x12ListActorTemplates\x12!.ateapi.ListActorTemplatesRequest\x1a\".ateapi.ListActorTemplatesResponse\"\x00\x12R\n\x13\x44\x65leteActorTemplate\x12\".ateapi.DeleteActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x32\x8a\x01\n\rActorIdentity\x12:\n\x07MintJWT\x12\x16.ateapi.MintJWTRequest\x1a\x17.ateapi.MintJWTResponse\x12=\n\x08MintCert\x12\x17.ateapi.MintCertRequest\x1a\x18.ateapi.MintCertResponse2i\n\rWorkerService\x12X\n\x11SetWorkerCapacity\x12 .ateapi.SetWorkerCapacityRequest\x1a!.ateapi.SetWorkerCapacityResponseB9Z7github.com/agent-substrate/substrate/pkg/proto/ateapipbb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -54,22 +54,22 @@ _globals['_EXTERNALVOLUME_VOLUMECONTEXTENTRY']._serialized_options = b'8\001' _globals['_WORKER_LABELSENTRY']._loaded_options = None _globals['_WORKER_LABELSENTRY']._serialized_options = b'8\001' - _globals['_SNAPSHOTCONTENTSCOPE']._serialized_start=9357 - _globals['_SNAPSHOTCONTENTSCOPE']._serialized_end=9485 - _globals['_ACTORSNAPSHOTTAGSCOPE']._serialized_start=9488 - _globals['_ACTORSNAPSHOTTAGSCOPE']._serialized_end=9632 - _globals['_ACTORSTATE']._serialized_start=9635 - _globals['_ACTORSTATE']._serialized_end=9882 - _globals['_SANDBOXCLASS']._serialized_start=9884 - _globals['_SANDBOXCLASS']._serialized_end=9982 - _globals['_RESUMESOURCE']._serialized_start=9984 - _globals['_RESUMESOURCE']._serialized_end=10084 - _globals['_ACTORMETADATAFIELD']._serialized_start=10087 - _globals['_ACTORMETADATAFIELD']._serialized_end=10241 - _globals['_WORKERSTATE']._serialized_start=10243 - _globals['_WORKERSTATE']._serialized_end=10338 - _globals['_ACTORCERTIFICATEPURPOSE']._serialized_start=10340 - _globals['_ACTORCERTIFICATEPURPOSE']._serialized_end=10447 + _globals['_SNAPSHOTCONTENTSCOPE']._serialized_start=9913 + _globals['_SNAPSHOTCONTENTSCOPE']._serialized_end=10041 + _globals['_ACTORSNAPSHOTTAGSCOPE']._serialized_start=10044 + _globals['_ACTORSNAPSHOTTAGSCOPE']._serialized_end=10188 + _globals['_ACTORSTATE']._serialized_start=10191 + _globals['_ACTORSTATE']._serialized_end=10438 + _globals['_SANDBOXCLASS']._serialized_start=10440 + _globals['_SANDBOXCLASS']._serialized_end=10538 + _globals['_RESUMESOURCE']._serialized_start=10540 + _globals['_RESUMESOURCE']._serialized_end=10640 + _globals['_ACTORMETADATAFIELD']._serialized_start=10643 + _globals['_ACTORMETADATAFIELD']._serialized_end=10797 + _globals['_WORKERSTATE']._serialized_start=10799 + _globals['_WORKERSTATE']._serialized_end=10894 + _globals['_ACTORCERTIFICATEPURPOSE']._serialized_start=10896 + _globals['_ACTORCERTIFICATEPURPOSE']._serialized_end=11003 _globals['_LOCALSNAPSHOTINFO']._serialized_start=87 _globals['_LOCALSNAPSHOTINFO']._serialized_end=221 _globals['_SELECTOR']._serialized_start=223 @@ -226,44 +226,56 @@ _globals['_DELETEACTORSNAPSHOTTAGREQUEST']._serialized_end=7694 _globals['_DELETEOPTIONS']._serialized_start=7696 _globals['_DELETEOPTIONS']._serialized_end=7741 - _globals['_LISTWORKERSREQUEST']._serialized_start=7743 - _globals['_LISTWORKERSREQUEST']._serialized_end=7802 - _globals['_LISTWORKERSRESPONSE']._serialized_start=7804 - _globals['_LISTWORKERSRESPONSE']._serialized_end=7883 - _globals['_GETWORKERREQUEST']._serialized_start=7885 - _globals['_GETWORKERREQUEST']._serialized_end=7938 - _globals['_CREATEWORKERREQUEST']._serialized_start=7940 - _globals['_CREATEWORKERREQUEST']._serialized_end=7993 - _globals['_UPDATEWORKERREQUEST']._serialized_start=7995 - _globals['_UPDATEWORKERREQUEST']._serialized_end=8048 - _globals['_DELETEWORKERREQUEST']._serialized_start=8050 - _globals['_DELETEWORKERREQUEST']._serialized_end=8146 - _globals['_DRAINWORKERREQUEST']._serialized_start=8148 - _globals['_DRAINWORKERREQUEST']._serialized_end=8203 - _globals['_LISTACTORSREQUEST']._serialized_start=8205 - _globals['_LISTACTORSREQUEST']._serialized_end=8281 - _globals['_LISTACTORSRESPONSE']._serialized_start=8283 - _globals['_LISTACTORSRESPONSE']._serialized_end=8359 - _globals['_WORKER']._serialized_start=8362 - _globals['_WORKER']._serialized_end=8730 - _globals['_WORKER_LABELSENTRY']._serialized_start=8685 - _globals['_WORKER_LABELSENTRY']._serialized_end=8730 - _globals['_WORKERSTATUS']._serialized_start=8732 - _globals['_WORKERSTATUS']._serialized_end=8827 - _globals['_WORKERCAPACITY']._serialized_start=8829 - _globals['_WORKERCAPACITY']._serialized_end=8886 - _globals['_ACTORASSIGNMENT']._serialized_start=8888 - _globals['_ACTORASSIGNMENT']._serialized_end=9005 - _globals['_MINTJWTREQUEST']._serialized_start=9007 - _globals['_MINTJWTREQUEST']._serialized_end=9098 - _globals['_MINTJWTRESPONSE']._serialized_start=9100 - _globals['_MINTJWTRESPONSE']._serialized_end=9136 - _globals['_MINTCERTREQUEST']._serialized_start=9139 - _globals['_MINTCERTREQUEST']._serialized_end=9306 - _globals['_MINTCERTRESPONSE']._serialized_start=9308 - _globals['_MINTCERTRESPONSE']._serialized_end=9354 - _globals['_CONTROL']._serialized_start=10450 - _globals['_CONTROL']._serialized_end=12886 - _globals['_ACTORIDENTITY']._serialized_start=12889 - _globals['_ACTORIDENTITY']._serialized_end=13027 + _globals['_LISTWORKERACTORASSIGNMENTSREQUEST']._serialized_start=7743 + _globals['_LISTWORKERACTORASSIGNMENTSREQUEST']._serialized_end=7852 + _globals['_LISTWORKERACTORASSIGNMENTSRESPONSE']._serialized_start=7854 + _globals['_LISTWORKERACTORASSIGNMENTSRESPONSE']._serialized_end=7967 + _globals['_LISTWORKERSREQUEST']._serialized_start=7969 + _globals['_LISTWORKERSREQUEST']._serialized_end=8028 + _globals['_LISTWORKERSRESPONSE']._serialized_start=8030 + _globals['_LISTWORKERSRESPONSE']._serialized_end=8109 + _globals['_GETWORKERREQUEST']._serialized_start=8111 + _globals['_GETWORKERREQUEST']._serialized_end=8164 + _globals['_CREATEWORKERREQUEST']._serialized_start=8166 + _globals['_CREATEWORKERREQUEST']._serialized_end=8219 + _globals['_UPDATEWORKERREQUEST']._serialized_start=8221 + _globals['_UPDATEWORKERREQUEST']._serialized_end=8274 + _globals['_DELETEWORKERREQUEST']._serialized_start=8276 + _globals['_DELETEWORKERREQUEST']._serialized_end=8372 + _globals['_DRAINWORKERREQUEST']._serialized_start=8374 + _globals['_DRAINWORKERREQUEST']._serialized_end=8429 + _globals['_LISTACTORSREQUEST']._serialized_start=8431 + _globals['_LISTACTORSREQUEST']._serialized_end=8507 + _globals['_LISTACTORSRESPONSE']._serialized_start=8509 + _globals['_LISTACTORSRESPONSE']._serialized_end=8585 + _globals['_WORKER']._serialized_start=8588 + _globals['_WORKER']._serialized_end=8914 + _globals['_WORKER_LABELSENTRY']._serialized_start=8869 + _globals['_WORKER_LABELSENTRY']._serialized_end=8914 + _globals['_WORKERSTATUS']._serialized_start=8916 + _globals['_WORKERSTATUS']._serialized_end=9012 + _globals['_WORKERALLOCATION']._serialized_start=9014 + _globals['_WORKERALLOCATION']._serialized_end=9119 + _globals['_WORKERRESOURCES']._serialized_start=9121 + _globals['_WORKERRESOURCES']._serialized_end=9192 + _globals['_ACTORASSIGNMENT']._serialized_start=9195 + _globals['_ACTORASSIGNMENT']._serialized_end=9394 + _globals['_SETWORKERCAPACITYREQUEST']._serialized_start=9396 + _globals['_SETWORKERCAPACITYREQUEST']._serialized_end=9500 + _globals['_SETWORKERCAPACITYRESPONSE']._serialized_start=9502 + _globals['_SETWORKERCAPACITYRESPONSE']._serialized_end=9561 + _globals['_MINTJWTREQUEST']._serialized_start=9563 + _globals['_MINTJWTREQUEST']._serialized_end=9654 + _globals['_MINTJWTRESPONSE']._serialized_start=9656 + _globals['_MINTJWTRESPONSE']._serialized_end=9692 + _globals['_MINTCERTREQUEST']._serialized_start=9695 + _globals['_MINTCERTREQUEST']._serialized_end=9862 + _globals['_MINTCERTRESPONSE']._serialized_start=9864 + _globals['_MINTCERTRESPONSE']._serialized_end=9910 + _globals['_CONTROL']._serialized_start=11006 + _globals['_CONTROL']._serialized_end=13561 + _globals['_ACTORIDENTITY']._serialized_start=13564 + _globals['_ACTORIDENTITY']._serialized_end=13702 + _globals['_WORKERSERVICE']._serialized_start=13704 + _globals['_WORKERSERVICE']._serialized_end=13809 # @@protoc_insertion_point(module_scope) diff --git a/benchmarking/locust/common/ateapi_pb2_grpc.py b/benchmarking/locust/common/ateapi_pb2_grpc.py index 37a0047a23..701aa78a70 100644 --- a/benchmarking/locust/common/ateapi_pb2_grpc.py +++ b/benchmarking/locust/common/ateapi_pb2_grpc.py @@ -164,6 +164,11 @@ def __init__(self, channel): request_serializer=ateapi__pb2.DrainWorkerRequest.SerializeToString, response_deserializer=ateapi__pb2.Worker.FromString, _registered_method=True) + self.ListWorkerActorAssignments = channel.unary_unary( + '/ateapi.Control/ListWorkerActorAssignments', + request_serializer=ateapi__pb2.ListWorkerActorAssignmentsRequest.SerializeToString, + response_deserializer=ateapi__pb2.ListWorkerActorAssignmentsResponse.FromString, + _registered_method=True) self.ListActors = channel.unary_unary( '/ateapi.Control/ListActors', request_serializer=ateapi__pb2.ListActorsRequest.SerializeToString, @@ -382,6 +387,14 @@ def DrainWorker(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def ListWorkerActorAssignments(self, request, context): + """List the Actors a Worker hosts. A subresource of Worker rather than a field + on it, so GetWorker and ListWorkers cost the same whatever the occupancy. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def ListActors(self, request, context): """List Actors. """ @@ -562,6 +575,11 @@ def add_ControlServicer_to_server(servicer, server): request_deserializer=ateapi__pb2.DrainWorkerRequest.FromString, response_serializer=ateapi__pb2.Worker.SerializeToString, ), + 'ListWorkerActorAssignments': grpc.unary_unary_rpc_method_handler( + servicer.ListWorkerActorAssignments, + request_deserializer=ateapi__pb2.ListWorkerActorAssignmentsRequest.FromString, + response_serializer=ateapi__pb2.ListWorkerActorAssignmentsResponse.SerializeToString, + ), 'ListActors': grpc.unary_unary_rpc_method_handler( servicer.ListActors, request_deserializer=ateapi__pb2.ListActorsRequest.FromString, @@ -1240,6 +1258,33 @@ def DrainWorker(request, metadata, _registered_method=True) + @staticmethod + def ListWorkerActorAssignments(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/ateapi.Control/ListWorkerActorAssignments', + ateapi__pb2.ListWorkerActorAssignmentsRequest.SerializeToString, + ateapi__pb2.ListWorkerActorAssignmentsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def ListActors(request, target, @@ -1630,3 +1675,94 @@ def MintCert(request, timeout, metadata, _registered_method=True) + + +class WorkerServiceStub: + """WorkerService is how a Worker tells the control plane about itself. It is + separate from Control because the two have different callers and different + authorization: Control is the client-facing API, while these RPCs are served + only to an atelet, and only for the Workers on its own node. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.SetWorkerCapacity = channel.unary_unary( + '/ateapi.WorkerService/SetWorkerCapacity', + request_serializer=ateapi__pb2.SetWorkerCapacityRequest.SerializeToString, + response_deserializer=ateapi__pb2.SetWorkerCapacityResponse.FromString, + _registered_method=True) + + +class WorkerServiceServicer: + """WorkerService is how a Worker tells the control plane about itself. It is + separate from Control because the two have different callers and different + authorization: Control is the client-facing API, while these RPCs are served + only to an atelet, and only for the Workers on its own node. + """ + + def SetWorkerCapacity(self, request, context): + """SetWorkerCapacity records what a Worker can hold. Capacity is the Worker's + to report rather than the control plane's to infer: it is what the ateom + can actually supply, only its node can observe it, and a fleet may run + mixed ateom versions. + + atelet calls this with its own client certificate, as it does for + MintCert. Idempotent: re-sending the same capacity is not a write. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_WorkerServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'SetWorkerCapacity': grpc.unary_unary_rpc_method_handler( + servicer.SetWorkerCapacity, + request_deserializer=ateapi__pb2.SetWorkerCapacityRequest.FromString, + response_serializer=ateapi__pb2.SetWorkerCapacityResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'ateapi.WorkerService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + server.add_registered_method_handlers('ateapi.WorkerService', rpc_method_handlers) + + + # This class is part of an EXPERIMENTAL API. +class WorkerService: + """WorkerService is how a Worker tells the control plane about itself. It is + separate from Control because the two have different callers and different + authorization: Control is the client-facing API, while these RPCs are served + only to an atelet, and only for the Workers on its own node. + """ + + @staticmethod + def SetWorkerCapacity(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/ateapi.WorkerService/SetWorkerCapacity', + ateapi__pb2.SetWorkerCapacityRequest.SerializeToString, + ateapi__pb2.SetWorkerCapacityResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/cmd/ateapi/internal/actoridentity/actoridentity.go b/cmd/ateapi/internal/actoridentity/actoridentity.go index f337dedcf7..a27e420f4a 100644 --- a/cmd/ateapi/internal/actoridentity/actoridentity.go +++ b/cmd/ateapi/internal/actoridentity/actoridentity.go @@ -26,6 +26,7 @@ import ( "path" "time" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth" "github.com/agent-substrate/substrate/cmd/ateapi/internal/controlapi" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" @@ -37,8 +38,6 @@ import ( "github.com/agent-substrate/substrate/internal/substratex509" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/peer" "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/api/operation" "k8s.io/apimachinery/pkg/util/validation/field" @@ -73,19 +72,7 @@ func New(actorIdentityJWTIssuer string, actorIDJWTPool localjwtauthority.Pool, a } } -// The SPIFFE identity that atelet client certs carry, as minted by the -// podidentity signer (cmd/podcertcontroller/internal/podidentitysigner). -// -// These mirror the constants the atelet dialer verifies against in -// cmd/ateapi/internal/controlapi/dialer.go. They are duplicated rather than -// imported so that this package does not depend on controlapi for three -// strings; if a third pkg that need these constants appears, they should move to a shared package. -const ( - ateletTrustDomain = "cluster.local" - ateletNamespace = "ate-system" - ateletSA = "atelet" - actorCertificateLifetime = time.Hour -) +const actorCertificateLifetime = time.Hour func (s *Server) MintJWT(ctx context.Context, req *ateapipb.MintJWTRequest) (*ateapipb.MintJWTResponse, error) { caller, ok := principal.FromContext(ctx) @@ -136,7 +123,7 @@ func (s *Server) MintJWT(ctx context.Context, req *ateapipb.MintJWTRequest) (*at } func (s *Server) MintCert(ctx context.Context, req *ateapipb.MintCertRequest) (*ateapipb.MintCertResponse, error) { - caller, err := authenticateAtelet(ctx) + caller, err := ateletauth.Authenticate(ctx) if err != nil { return nil, err } @@ -154,8 +141,9 @@ func (s *Server) MintCert(ctx context.Context, req *ateapipb.MintCertRequest) (* } atespace, actorName := actorRef.Atespace, actorRef.Name - // Actor identity comes only from ateapi state. expected_actor_uid is a - // fail-closed guard against a request crossing an assignment change. + // expected_actor_uid picked which actor to mint for (see authorizeActor); + // re-checking it here fails closed if the request crossed an assignment + // change. actorUID := actor.GetMetadata().GetUid() if actorUID == "" { slog.ErrorContext(ctx, "MintCert: actor has no UID", slog.Any("actor", actorRef)) @@ -218,62 +206,6 @@ func (s *Server) MintCert(ctx context.Context, req *ateapipb.MintCertRequest) (* }, nil } -// ateletCaller is the verified identity of an atelet requesting an actor credential. -type ateletCaller struct { - podName string - nodeName string -} - -// authenticateAtelet verifies that the RPC arrived over mTLS from an atelet, -// and returns the identity that atelet's certificate asserts. -// -// The certificate chain is already verified by the TLS layer against the -// pod-identity CA (see buildServerCreds in cmd/ateapi/main.go), so the -// extensions read here are trustworthy: only the pod-identity signer can mint -// a certificate carrying a given pod's node name. -func authenticateAtelet(ctx context.Context) (*ateletCaller, error) { - p, ok := peer.FromContext(ctx) - if !ok { - return nil, status.Errorf(codes.Unauthenticated, "no peer transport information found") - } - - tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) - if !ok { - return nil, status.Errorf(codes.Unauthenticated, "unexpected peer transport credentials") - } - - if len(tlsInfo.State.PeerCertificates) == 0 { - return nil, status.Errorf(codes.Unauthenticated, "could not verify peer certificate") - } - leaf := tlsInfo.State.PeerCertificates[0] - - // Only atelet may mint actor credentials. Everything else with a valid - // pod-identity certificate — including the actor workloads themselves — is - // rejected here. - expected := (&url.URL{ - Scheme: "spiffe", - Host: ateletTrustDomain, - Path: path.Join("ns", ateletNamespace, "sa", ateletSA), - }).String() - if len(leaf.URIs) == 0 || leaf.URIs[0].String() != expected { - slog.WarnContext(ctx, "ActorIdentity denied: caller is not atelet", - slog.Any("uris", leaf.URIs), slog.String("expected", expected)) - return nil, status.Errorf(codes.PermissionDenied, "caller is not permitted to mint actor credentials") - } - - identity, err := substratex509.PodIdentityFromCertificate(leaf) - if err != nil { - slog.WarnContext(ctx, "ActorIdentity denied: malformed PodIdentity extension", slog.Any("err", err)) - return nil, status.Errorf(codes.PermissionDenied, "caller is not permitted to mint actor credentials") - } - if identity == nil { - slog.WarnContext(ctx, "ActorIdentity denied: certificate has no PodIdentity extension") - return nil, status.Errorf(codes.PermissionDenied, "caller is not permitted to mint actor credentials") - } - - return &ateletCaller{podName: identity.PodName, nodeName: identity.NodeName}, nil -} - func validateMintJWTRequest(ctx context.Context, req *ateapipb.MintJWTRequest) field.ErrorList { // Call the generated validation. op := operation.Operation{Type: operation.Create} @@ -286,13 +218,14 @@ func validateMintCertRequest(ctx context.Context, req *ateapipb.MintCertRequest) return controlapi.Validate_MintCertRequest(ctx, op, nil, req, nil) } -// authorizeActor resolves the actor from the authenticated worker and verifies -// that the worker and actor still point at one another. Actor identity supplied -// by the requester never participates in this authorization decision. +// authorizeActor resolves the actor the request names among those the worker is +// hosting and verifies the two still point at one another. Requester-supplied +// identity never participates in the decision. +// // The worker is resolved from cache first (hot path), but cache misses and // denials fall back to the authoritative store to handle watch-delivery lag // right after ResumeActor. -func (s *Server) authorizeActor(ctx context.Context, caller *ateletCaller, req *ateapipb.MintCertRequest) (*ateapipb.Actor, resources.ActorRef, error) { +func (s *Server) authorizeActor(ctx context.Context, caller *ateletauth.Caller, req *ateapipb.MintCertRequest) (*ateapipb.Actor, resources.ActorRef, error) { reason := "worker not found" worker, err := s.workers.Worker(req.GetWorker().GetName()) if err != nil && !errors.Is(err, store.ErrNotFound) { @@ -337,9 +270,9 @@ func (s *Server) authorizeActor(ctx context.Context, caller *ateletCaller, req * // denyMint logs the internal reason and returns a uniform PermissionDenied. // Denials are deliberately indistinguishable from each other: a caller that // is not entitled to a worker should not learn its assignment. -func (s *Server) denyMint(ctx context.Context, caller *ateletCaller, req *ateapipb.MintCertRequest, reason string, args ...any) error { +func (s *Server) denyMint(ctx context.Context, caller *ateletauth.Caller, req *ateapipb.MintCertRequest, reason string, args ...any) error { slog.WarnContext(ctx, "ActorIdentity denied: "+reason, - append([]any{slog.String("worker", req.GetWorker().GetName()), slog.String("callerPod", caller.podName), slog.String("callerNode", caller.nodeName)}, args...)...) + append([]any{slog.String("worker", req.GetWorker().GetName()), slog.String("callerPod", caller.PodName), slog.String("callerNode", caller.NodeName)}, args...)...) return status.Error(codes.PermissionDenied, "caller is not permitted to mint credentials for this actor") } @@ -347,14 +280,22 @@ var errAssignmentMismatch = errors.New("assignment mismatch") // authorizeWithWorker returns errAssignmentMismatch and a reason string if the authorization failed // due to an assignment mismatch, indicating the caller may want to refetch the worker and retry. -func (s *Server) authorizeWithWorker(ctx context.Context, worker *ateapipb.Worker, caller *ateletCaller, req *ateapipb.MintCertRequest) (*ateapipb.Actor, resources.ActorRef, string, error) { - if worker.GetNodeName() != caller.nodeName { +func (s *Server) authorizeWithWorker(ctx context.Context, worker *ateapipb.Worker, caller *ateletauth.Caller, req *ateapipb.MintCertRequest) (*ateapipb.Actor, resources.ActorRef, string, error) { + if worker.GetNodeName() != caller.NodeName { return nil, resources.ActorRef{}, "worker is hosted on a different node", errAssignmentMismatch } - actorRef := resources.ActorRefFromObjectRef(worker.GetStatus().GetAssignment().GetActor()) + assigned, err := s.assignmentToMintFor(ctx, worker.GetMetadata().GetName(), req.GetExpectedActorUid()) + if errors.Is(err, store.ErrNotFound) { + return nil, resources.ActorRef{}, "worker is not hosting the requested actor", errAssignmentMismatch + } + if err != nil { + slog.ErrorContext(ctx, "ActorIdentity: failed to read worker assignment", slog.Any("err", err)) + return nil, resources.ActorRef{}, "", status.Error(codes.Internal, "failed to look up worker assignment") + } + actorRef := resources.ActorRefFromObjectRef(assigned.GetActor()) if actorRef == (resources.ActorRef{}) { - return nil, resources.ActorRef{}, "worker has no actor assignment", errAssignmentMismatch + return nil, resources.ActorRef{}, "worker assignment names no actor", errAssignmentMismatch } actor, err := s.store.GetActor(ctx, actorRef) @@ -378,7 +319,7 @@ func (s *Server) authorizeWithWorker(ctx context.Context, worker *ateapipb.Worke slog.ErrorContext(ctx, "ActorIdentity: running actor has no worker assignment", slog.Any("actor", actorRef)) return nil, resources.ActorRef{}, "", status.Error(codes.FailedPrecondition, "actor has no worker assigned") } - if worker.GetStatus().GetAssignment().GetActorUid() != actor.GetMetadata().GetUid() { + if assigned.GetActorUid() != actor.GetMetadata().GetUid() { return nil, resources.ActorRef{}, "worker is no longer assigned to this actor incarnation", errAssignmentMismatch } if assignment.GetWorker().GetName() != worker.GetMetadata().GetName() { @@ -386,3 +327,30 @@ func (s *Server) authorizeWithWorker(ctx context.Context, worker *ateapipb.Worke } return actor, actorRef, "", nil } + +// assignmentToMintFor picks which of the worker's actors to mint for, or +// ErrNotFound when it hosts none. expected_actor_uid selects from what ateapi +// records the worker as hosting; it does not assert. +// +// Falling back to another of the worker's assignments keeps a bad binding +// (PermissionDenied) apart from a stale expectation (retryable). Reads go to +// the store: the binding was committed moments ago and the watch has not +// delivered it. Only one other assignment is needed to tell the two apart, so +// the fallback reads a single row rather than the Worker's whole occupancy. +func (s *Server) assignmentToMintFor(ctx context.Context, workerName, actorUID string) (*ateapipb.ActorAssignment, error) { + assigned, err := s.store.GetWorkerAssignment(ctx, workerName, actorUID) + if err == nil { + return assigned, nil + } + if !errors.Is(err, store.ErrNotFound) { + return nil, err + } + page, err := s.store.ListWorkerAssignments(ctx, workerName, store.ListOptions{PageSize: 1}) + if err != nil { + return nil, err + } + if len(page.Items) == 0 { + return nil, store.ErrNotFound + } + return page.Items[0], nil +} diff --git a/cmd/ateapi/internal/actoridentity/actoridentity_test.go b/cmd/ateapi/internal/actoridentity/actoridentity_test.go index e4b9db28c9..5df75e0c25 100644 --- a/cmd/ateapi/internal/actoridentity/actoridentity_test.go +++ b/cmd/ateapi/internal/actoridentity/actoridentity_test.go @@ -18,17 +18,17 @@ import ( "context" "crypto/ed25519" "crypto/rand" - "crypto/tls" "crypto/x509" "crypto/x509/pkix" "fmt" - "math/big" - "net/url" "path" "strings" "testing" "time" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" @@ -39,8 +39,6 @@ import ( "github.com/agent-substrate/substrate/internal/substratex509" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/peer" "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -75,73 +73,6 @@ const ( // populates. Self-signing is sufficient because the code under test reads an // already transport-verified peer certificate and never re-validates the chain // itself. -func newTestCert(t *testing.T, spiffePath string, podIdentity *substratex509.PodIdentity) *x509.Certificate { - t.Helper() - - pub, priv, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - t.Fatalf("generate key: %v", err) - } - - template := &x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{CommonName: "test-caller"}, - NotBefore: time.Now().Add(-time.Hour), - NotAfter: time.Now().Add(time.Hour), - } - if spiffePath != "" { - template.URIs = []*url.URL{{Scheme: "spiffe", Host: ateletTrustDomain, Path: spiffePath}} - } - if podIdentity != nil { - if err := substratex509.AddPodIdentityToCertificate(podIdentity, template); err != nil { - t.Fatalf("add pod identity: %v", err) - } - } - - der, err := x509.CreateCertificate(rand.Reader, template, template, pub, priv) - if err != nil { - t.Fatalf("create certificate: %v", err) - } - cert, err := x509.ParseCertificate(der) - if err != nil { - t.Fatalf("parse certificate: %v", err) - } - return cert -} - -// podIdentityOn returns a well-formed atelet PodIdentity pinned to nodeName. -func podIdentityOn(nodeName string) *substratex509.PodIdentity { - return &substratex509.PodIdentity{ - Namespace: ateletNamespace, - ServiceAccountName: ateletSA, - ServiceAccountUID: "sa-uid", - PodName: "atelet-xyz", - PodUID: "pod-uid", - NodeName: nodeName, - NodeUID: "node-uid", - } -} - -// ateletCertOn returns the certificate of the atelet running on nodeName. -func ateletCertOn(t *testing.T, nodeName string) *x509.Certificate { - t.Helper() - return newTestCert(t, path.Join("ns", ateletNamespace, "sa", ateletSA), podIdentityOn(nodeName)) -} - -// ctxWithCert injects cert as the transport-authenticated peer certificate. -// A nil cert yields a context with no peer information at all, which is what -// an unauthenticated call looks like. -func ctxWithCert(cert *x509.Certificate) context.Context { - ctx := context.Background() - if cert == nil { - return ctx - } - return peer.NewContext(ctx, &peer.Peer{ - AuthInfo: credentials.TLSInfo{ - State: tls.ConnectionState{PeerCertificates: []*x509.Certificate{cert}}, - }, - }) -} // newTestServer returns a Server backed by st, with a freshly generated actor // CA pool written to a temp file. @@ -220,27 +151,14 @@ func TestMintCertReadsThroughStaleWorkerCache(t *testing.T) { if assignInStore { // Phase 2: commit the assignment to the store only, as // AssignWorker does (possibly on another replica). - worker, err := st.GetWorker(ctx, testWorkerName) - if err != nil { - t.Fatalf("read seeded worker: %v", err) - } - _, err = st.UpdateWorker(ctx, testWorkerName, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { - if toUpdate.Status == nil { - toUpdate.Status = &ateapipb.WorkerStatus{} - } - toUpdate.Status.Assignment = &ateapipb.ActorAssignment{ - Actor: (resources.ActorRef{Atespace: testAtespace, Name: testActorName}).ToObjectRef(), - ActorUid: actor.GetMetadata().GetUid(), - } - return nil + bindActor(t, ctx, st, testWorkerName, &ateapipb.ActorAssignment{ + Actor: (resources.ActorRef{Atespace: testAtespace, Name: testActorName}).ToObjectRef(), + ActorUid: actor.GetMetadata().GetUid(), }) - if err != nil { - t.Fatalf("assign worker in store: %v", err) - } } srv := newTestServerWithCache(t, st, workers) - resp, err := srv.MintCert(ctxWithCert(ateletCertOn(t, testNode)), mintCertRequest(t, actor.GetMetadata().GetUid())) + resp, err := srv.MintCert(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)), mintCertRequest(t, actor.GetMetadata().GetUid())) wantCode := codes.PermissionDenied if assignInStore { @@ -256,6 +174,71 @@ func TestMintCertReadsThroughStaleWorkerCache(t *testing.T) { } } +// The multi-actor case of the same lag: the cached worker is not unassigned but +// hosting somebody else, so the requested actor's absence looks like an answer +// rather than a miss. +func TestMintCertReadsThroughForAnActorTheCacheHasNotSeenYet(t *testing.T) { + ctx := context.Background() + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + + // The cache seeds with the worker hosting only the first actor, and (via + // the inert watch) never learns of the second. + seedActor(t, ctx, st, actorFixture{state: ateapipb.ActorState_ACTOR_STATE_RUNNING, workerNode: testNode}) + workers := workercache.New(staleWatchStore{st}, time.Hour) + cacheCtx, cancel := context.WithCancel(ctx) + t.Cleanup(cancel) + if err := workers.Start(cacheCtx); err != nil { + t.Fatalf("start worker cache: %v", err) + } + + const secondActorName = "counter-2" + second, err := st.CreateActor(ctx, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: secondActorName}, + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_RUNNING, + WorkerAssignment: &ateapipb.WorkerAssignment{ + Worker: &ateapipb.ObjectRef{Name: testWorkerName}, + WorkerNamespace: testPodNS, + WorkerPool: testPool, + WorkerPod: testWorkerPod, + WorkerPodUid: testWorkerPodUID, + }, + }, + ActorTemplate: &ateapipb.ObjectRef{Atespace: "ate-demo", Name: "counter"}, + }) + if err != nil { + t.Fatalf("seed second actor: %v", err) + } + + // Bind it to the worker in the store only, as AssignWorker does. + bindActor(t, ctx, st, testWorkerName, &ateapipb.ActorAssignment{ + Actor: (resources.ActorRef{Atespace: testAtespace, Name: secondActorName}).ToObjectRef(), + ActorUid: second.GetMetadata().GetUid(), + }) + + srv := newTestServerWithCache(t, st, workers) + resp, err := srv.MintCert(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)), mintCertRequest(t, second.GetMetadata().GetUid())) + if err != nil { + t.Fatalf("MintCert() for an actor the cache has not seen: %v", err) + } + if len(resp.GetActorCertificates()) == 0 { + t.Fatal("MintCert() returned no certificates") + } + + leaf, err := x509.ParseCertificate(resp.GetActorCertificates()[0]) + if err != nil { + t.Fatalf("parse minted certificate: %v", err) + } + identity, err := substratex509.ActorIdentityFromCertificate(leaf) + if err != nil { + t.Fatalf("ActorIdentityFromCertificate: %v", err) + } + if identity == nil || identity.ActorName != secondActorName { + t.Errorf("minted for %+v, want actor %q", identity, secondActorName) + } +} + // TestMintCertReadsThroughWorkerCacheMiss pins the read-through for a worker // the cache has never seen: a worker registered moments before assignment may // be committed to the store (possibly by another replica) before this @@ -295,20 +278,18 @@ func TestMintCertReadsThroughWorkerCacheMiss(t *testing.T) { WorkerPod: testWorkerPod, WorkerPodUid: testWorkerPodUID, NodeName: testNode, - Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Assignment: &ateapipb.ActorAssignment{ - Actor: (resources.ActorRef{Atespace: testAtespace, Name: testActorName}).ToObjectRef(), - ActorUid: actor.GetMetadata().GetUid(), - }, - }, + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE}, }); err != nil { t.Fatalf("register worker in store: %v", err) } + bindActor(t, ctx, st, testWorkerName, &ateapipb.ActorAssignment{ + Actor: (resources.ActorRef{Atespace: testAtespace, Name: testActorName}).ToObjectRef(), + ActorUid: actor.GetMetadata().GetUid(), + }) } srv := newTestServerWithCache(t, st, workers) - resp, err := srv.MintCert(ctxWithCert(ateletCertOn(t, testNode)), mintCertRequest(t, actor.GetMetadata().GetUid())) + resp, err := srv.MintCert(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)), mintCertRequest(t, actor.GetMetadata().GetUid())) wantCode := codes.PermissionDenied if workerInStore { @@ -470,20 +451,26 @@ func seedActor(t *testing.T, ctx context.Context, st store.Interface, f actorFix WorkerPod: testWorkerPod, WorkerPodUid: testWorkerPodUID, NodeName: f.workerNode, - Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Assignment: &ateapipb.ActorAssignment{ - Actor: assigned.ToObjectRef(), - ActorUid: assignedActorUID, - }, - }, - } - if f.unassigned { - worker.Status.Assignment = nil + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE}, } if _, err := st.CreateWorker(ctx, worker); err != nil { t.Fatalf("seed worker: %v", err) } + if f.unassigned { + return + } + bindActor(t, ctx, st, testWorkerName, &ateapipb.ActorAssignment{ + Actor: assigned.ToObjectRef(), + ActorUid: assignedActorUID, + }) +} + +// bindActor places an actor on a worker at whatever version it is currently at. +func bindActor(t *testing.T, ctx context.Context, st store.Interface, workerName string, assignment *ateapipb.ActorAssignment) { + t.Helper() + if err := st.BindActorToWorker(ctx, workerName, assignment, nil); err != nil { + t.Fatalf("bind actor %s to worker: %v", assignment.GetActorUid(), err) + } } // runningOnNode is the fixture for a healthy actor hosted on nodeName. @@ -530,32 +517,32 @@ func TestMintCertAuthorization(t *testing.T) { }, "caller is not the atelet service account": { cert: func(t *testing.T) *x509.Certificate { - id := podIdentityOn(testNode) + id := ateletauthtest.PodIdentityOn(testNode) id.ServiceAccountName = "some-workload" - return newTestCert(t, path.Join("ns", ateletNamespace, "sa", "some-workload"), id) + return ateletauthtest.Cert(t, path.Join("ns", ateletauth.Namespace, "sa", "some-workload"), id) }, fixture: runningOnNode(testNode), wantCode: codes.PermissionDenied, }, "caller is an atelet in the wrong namespace": { cert: func(t *testing.T) *x509.Certificate { - id := podIdentityOn(testNode) + id := ateletauthtest.PodIdentityOn(testNode) id.Namespace = "someone-elses-system" - return newTestCert(t, path.Join("ns", "someone-elses-system", "sa", ateletSA), id) + return ateletauthtest.Cert(t, path.Join("ns", "someone-elses-system", "sa", ateletauth.ServiceAccount), id) }, fixture: runningOnNode(testNode), wantCode: codes.PermissionDenied, }, "certificate carries no SPIFFE URI": { cert: func(t *testing.T) *x509.Certificate { - return newTestCert(t, "", podIdentityOn(testNode)) + return ateletauthtest.Cert(t, "", ateletauthtest.PodIdentityOn(testNode)) }, fixture: runningOnNode(testNode), wantCode: codes.PermissionDenied, }, "certificate carries no PodIdentity extension": { cert: func(t *testing.T) *x509.Certificate { - return newTestCert(t, path.Join("ns", ateletNamespace, "sa", ateletSA), nil) + return ateletauthtest.Cert(t, path.Join("ns", ateletauth.Namespace, "sa", ateletauth.ServiceAccount), nil) }, fixture: runningOnNode(testNode), wantCode: codes.PermissionDenied, @@ -668,7 +655,7 @@ func TestMintCertAuthorization(t *testing.T) { case tc.cert != nil: callerCert = tc.cert(t) default: - callerCert = ateletCertOn(t, testNode) + callerCert = ateletauthtest.CertOn(t, testNode) } actor, err := st.GetActor(ctx, resources.ActorRef{Atespace: testAtespace, Name: testActorName}) @@ -685,7 +672,7 @@ func TestMintCertAuthorization(t *testing.T) { if tc.expectedActorUID != nil { req.ExpectedActorUid = *tc.expectedActorUID } - resp, err := srv.MintCert(ctxWithCert(callerCert), req) + resp, err := srv.MintCert(ateletauthtest.ContextWith(callerCert), req) if got := status.Code(err); got != tc.wantCode { t.Fatalf("MintCert() code = %v (err = %v), want %v", got, err, tc.wantCode) } @@ -694,7 +681,7 @@ func TestMintCertAuthorization(t *testing.T) { // message must not vary with why the mint was refused, or a // caller could probe workers it is not entitled to. msg := status.Convert(err).Message() - if msg != "caller is not permitted to mint actor credentials" && + if msg != "caller is not permitted" && msg != "caller is not permitted to mint credentials for this actor" { t.Errorf("MintCert() denial leaks its reason: %q", msg) } @@ -728,7 +715,7 @@ func TestMintCertRejectsUnsupportedPurpose(t *testing.T) { "unknown": ateapipb.ActorCertificatePurpose(99), } { t.Run(name, func(t *testing.T) { - _, err := server.MintCert(ctxWithCert(ateletCertOn(t, testNode)), &ateapipb.MintCertRequest{Purpose: purpose}) + _, err := server.MintCert(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)), &ateapipb.MintCertRequest{Purpose: purpose}) if got := status.Code(err); got != codes.InvalidArgument { t.Fatalf("MintCert() code = %v (err = %v), want %v", got, err, codes.InvalidArgument) } @@ -756,7 +743,7 @@ func mintCertFor(t *testing.T, request func(actorUID string) *ateapipb.MintCertR t.Fatal("seeded actor has no UID; the store is expected to assign one") } - resp, err := newTestServer(t, st).MintCert(ctxWithCert(ateletCertOn(t, testNode)), request(actorUID)) + resp, err := newTestServer(t, st).MintCert(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)), request(actorUID)) if err != nil { return nil, actorUID, err } @@ -869,7 +856,7 @@ func TestMintCertActorState(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = srv.MintCert(ctxWithCert(ateletCertOn(t, testNode)), mintCertRequest(t, actor.GetMetadata().GetUid())) + _, err = srv.MintCert(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)), mintCertRequest(t, actor.GetMetadata().GetUid())) if got := status.Code(err); got != wantCode { t.Errorf("MintCert() code = %v (err = %v), want %v", got, err, wantCode) } @@ -904,7 +891,7 @@ func TestMintCertDeniesUnassignedActorWhateverItsState(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = srv.MintCert(ctxWithCert(ateletCertOn(t, testNode)), mintCertRequest(t, actor.GetMetadata().GetUid())) + _, err = srv.MintCert(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)), mintCertRequest(t, actor.GetMetadata().GetUid())) if got := status.Code(err); got != codes.PermissionDenied { t.Errorf("MintCert() code = %v (err = %v), want %v", got, err, codes.PermissionDenied) } @@ -958,7 +945,7 @@ func TestMintCertAuthorizesBeforeSigning(t *testing.T) { } req := mintCertRequest(t, actor.GetMetadata().GetUid()) req.CertificateSigningRequest = []byte("not a CSR") - _, err = srv.MintCert(ctxWithCert(ateletCertOn(t, testNode)), req) + _, err = srv.MintCert(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, testNode)), req) if got := status.Code(err); got != codes.PermissionDenied { t.Errorf("MintCert() code = %v (err = %v), want %v", got, err, codes.PermissionDenied) } diff --git a/cmd/ateapi/internal/ateletauth/ateletauth.go b/cmd/ateapi/internal/ateletauth/ateletauth.go new file mode 100644 index 0000000000..70eb7e0cee --- /dev/null +++ b/cmd/ateapi/internal/ateletauth/ateletauth.go @@ -0,0 +1,104 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ateletauth authenticates RPCs that arrive from an atelet, for the +// ateapi services served only to atelet. +package ateletauth + +import ( + "context" + "log/slog" + "net/url" + "path" + + "github.com/agent-substrate/substrate/internal/substratex509" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +// The SPIFFE identity that atelet client certs carry, as minted by the +// podidentity signer (cmd/podcertcontroller/internal/podidentitysigner). +// +// These mirror the constants the atelet dialer verifies against in +// cmd/ateapi/internal/controlapi/dialer.go, duplicated rather than imported so +// that this package does not depend on controlapi for three strings. +const ( + TrustDomain = "cluster.local" + Namespace = "ate-system" + ServiceAccount = "atelet" +) + +// Caller is the verified identity of an atelet. +type Caller struct { + PodName string + NodeName string +} + +// Authenticate verifies that the RPC arrived over mTLS from an atelet, and +// returns the identity that atelet's certificate asserts. +// +// The certificate chain is already verified by the TLS layer against the +// pod-identity CA (see buildServerCreds in cmd/ateapi/main.go), so the +// extensions read here are trustworthy: only the pod-identity signer can mint +// a certificate carrying a given pod's node name. +func Authenticate(ctx context.Context) (*Caller, error) { + p, ok := peer.FromContext(ctx) + if !ok { + return nil, status.Errorf(codes.Unauthenticated, "no peer transport information found") + } + + tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) + if !ok { + return nil, status.Errorf(codes.Unauthenticated, "unexpected peer transport credentials") + } + + if len(tlsInfo.State.PeerCertificates) == 0 { + return nil, status.Errorf(codes.Unauthenticated, "could not verify peer certificate") + } + leaf := tlsInfo.State.PeerCertificates[0] + + // Only atelet may call these RPCs. Everything else with a valid + // pod-identity certificate — including the actor workloads themselves — is + // rejected here. + expected := (&url.URL{ + Scheme: "spiffe", + Host: TrustDomain, + Path: path.Join("ns", Namespace, "sa", ServiceAccount), + }).String() + if len(leaf.URIs) == 0 || leaf.URIs[0].String() != expected { + slog.WarnContext(ctx, "Denied: caller is not atelet", + slog.Any("uris", leaf.URIs), slog.String("expected", expected)) + return nil, denied() + } + + identity, err := substratex509.PodIdentityFromCertificate(leaf) + if err != nil { + slog.WarnContext(ctx, "Denied: malformed PodIdentity extension", slog.Any("err", err)) + return nil, denied() + } + if identity == nil { + slog.WarnContext(ctx, "Denied: certificate has no PodIdentity extension") + return nil, denied() + } + + return &Caller{PodName: identity.PodName, NodeName: identity.NodeName}, nil +} + +// denied is deliberately uniform: a caller learns that it is not atelet, and +// nothing about why. +func denied() error { + return status.Error(codes.PermissionDenied, "caller is not permitted") +} diff --git a/cmd/ateapi/internal/ateletauth/ateletauthtest/ateletauthtest.go b/cmd/ateapi/internal/ateletauth/ateletauthtest/ateletauthtest.go new file mode 100644 index 0000000000..0d258bb08a --- /dev/null +++ b/cmd/ateapi/internal/ateletauth/ateletauthtest/ateletauthtest.go @@ -0,0 +1,107 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ateletauthtest builds the peer contexts that the ateapi services +// served only to atelet authenticate against. +package ateletauthtest + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net/url" + "path" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth" + "github.com/agent-substrate/substrate/internal/substratex509" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" +) + +// Cert returns a self-signed certificate carrying the given SPIFFE path and +// PodIdentity, either of which may be omitted to produce a certificate that +// authentication must reject. +func Cert(t *testing.T, spiffePath string, podIdentity *substratex509.PodIdentity) *x509.Certificate { + t.Helper() + + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-caller"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + } + if spiffePath != "" { + template.URIs = []*url.URL{{Scheme: "spiffe", Host: ateletauth.TrustDomain, Path: spiffePath}} + } + if podIdentity != nil { + if err := substratex509.AddPodIdentityToCertificate(podIdentity, template); err != nil { + t.Fatalf("add pod identity: %v", err) + } + } + + der, err := x509.CreateCertificate(rand.Reader, template, template, pub, priv) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + cert, err := x509.ParseCertificate(der) + if err != nil { + t.Fatalf("parse certificate: %v", err) + } + return cert +} + +// PodIdentityOn returns a well-formed atelet PodIdentity pinned to nodeName. +func PodIdentityOn(nodeName string) *substratex509.PodIdentity { + return &substratex509.PodIdentity{ + Namespace: ateletauth.Namespace, + ServiceAccountName: ateletauth.ServiceAccount, + ServiceAccountUID: "sa-uid", + PodName: "atelet-xyz", + PodUID: "pod-uid", + NodeName: nodeName, + NodeUID: "node-uid", + } +} + +// CertOn returns the certificate of the atelet running on nodeName. +func CertOn(t *testing.T, nodeName string) *x509.Certificate { + t.Helper() + return Cert(t, path.Join("ns", ateletauth.Namespace, "sa", ateletauth.ServiceAccount), PodIdentityOn(nodeName)) +} + +// ContextWith injects cert as the transport-authenticated peer certificate. A +// nil cert yields a context with no peer information at all, which is what an +// unauthenticated call looks like. +func ContextWith(cert *x509.Certificate) context.Context { + ctx := context.Background() + if cert == nil { + return ctx + } + return peer.NewContext(ctx, &peer.Peer{ + AuthInfo: credentials.TLSInfo{ + State: tls.ConnectionState{PeerCertificates: []*x509.Certificate{cert}}, + }, + }) +} diff --git a/cmd/ateapi/internal/controlapi/common_test.go b/cmd/ateapi/internal/controlapi/common_test.go index 26daa92ac7..186556583b 100644 --- a/cmd/ateapi/internal/controlapi/common_test.go +++ b/cmd/ateapi/internal/controlapi/common_test.go @@ -15,9 +15,11 @@ package controlapi import ( + "context" "fmt" "testing" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/protobuf/testing/protocmp" "k8s.io/apimachinery/pkg/util/validation/field" @@ -46,3 +48,33 @@ func assertValidateErr(t *testing.T, got field.ErrorList, want field.ErrorList) t.Helper() field.ErrorMatcher{}.ByType().ByField().ByOrigin().Test(t, want, got) } + +// firstAssignment returns the single Actor a Worker is hosting, or nil when it +// is hosting none. These tests place one Actor per Worker, so "the assignment" +// is still a meaningful thing to assert on even though a Worker holds a set; +// asserting through this keeps them readable and would fail loudly (by looking +// at the wrong entry) if a test ever placed two. +func firstAssignment(t *testing.T, st store.Interface, workerName string) *ateapipb.ActorAssignment { + t.Helper() + page, err := st.ListWorkerAssignments(context.Background(), workerName, store.ListOptions{}) + if err != nil { + t.Fatalf("list assignments of worker %q: %v", workerName, err) + } + if len(page.Items) == 0 { + return nil + } + return page.Items[0] +} + +// seedAssignment places an actor on an already-created worker, which is how a +// test arranges a worker that is already hosting something. +func seedAssignment(t *testing.T, st store.Interface, workerName string, assignment *ateapipb.ActorAssignment) { + t.Helper() + if assignment == nil { + return + } + ctx := context.Background() + if err := st.BindActorToWorker(ctx, workerName, assignment, nil); err != nil { + t.Fatalf("seed assignment on worker %q: %v", workerName, err) + } +} diff --git a/cmd/ateapi/internal/controlapi/crash.go b/cmd/ateapi/internal/controlapi/crash.go index b4bffdbec1..8419d7f7a5 100644 --- a/cmd/ateapi/internal/controlapi/crash.go +++ b/cmd/ateapi/internal/controlapi/crash.go @@ -73,7 +73,7 @@ func crashActor(ctx context.Context, st crashActorStore, actorRef resources.Acto // instead leaves the actor (and its assignment) intact so the caller retries // crashActor, which re-attempts the release. releaseWorker is idempotent, so // a retry after a release that already succeeded is a no-op. - sandboxClass, err := releaseWorker(ctx, st, actor) + sandboxClass, _, err := releaseWorker(ctx, st, actor) if err != nil { return fmt.Errorf("while releasing worker to crash actor: %w", err) } @@ -109,17 +109,18 @@ type crashActorStore interface { GetActor(ctx context.Context, actorRef resources.ActorRef) (*ateapipb.Actor, error) UpdateActor(ctx context.Context, actorRef resources.ActorRef, precondition store.Precondition, mutate func(toUpdate *ateapipb.Actor) error) (*ateapipb.Actor, error) GetWorker(ctx context.Context, name string) (*ateapipb.Worker, error) - UpdateWorker(ctx context.Context, name string, precondition store.Precondition, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) + ReleaseActorFromWorker(ctx context.Context, workerName string, actorUID string) (*ateapipb.Worker, error) } // releaseWorker clears the worker's assignment if it still points at the given // actor. A missing worker or an already-cleared assignment is not an error. -// It returns the worker's sandboxClass if found. -func releaseWorker(ctx context.Context, st crashActorStore, actor *ateapipb.Actor) (string, error) { +// It returns the worker's sandboxClass if found, and the worker as it stands +// after the release, which callers holding a cache of workers hand to it. +func releaseWorker(ctx context.Context, st crashActorStore, actor *ateapipb.Actor) (string, *ateapipb.Worker, error) { assignment := actor.GetStatus().GetWorkerAssignment() if assignment == nil { slog.WarnContext(ctx, "Actor's worker assignment is already cleared") - return "", nil + return "", nil, nil } workerName := assignment.GetWorker().GetName() @@ -127,29 +128,23 @@ func releaseWorker(ctx context.Context, st crashActorStore, actor *ateapipb.Acto if errors.Is(err, store.ErrNotFound) { // No need to release if the worker is not found. slog.WarnContext(ctx, "Worker already gone while crashing actor, skipping release", slog.String("worker", workerName)) - return "", nil + return "", nil, nil } if err != nil { - return "", fmt.Errorf("while getting worker to release: %w", err) + return "", nil, fmt.Errorf("while getting worker to release: %w", err) } sandboxClass := worker.GetSandboxClass() - wass := worker.GetStatus().GetAssignment() - if wass == nil { - slog.WarnContext(ctx, "Worker's assignment is already nil, skipping release", slog.String("worker", workerName)) - return sandboxClass, nil - } - // Only free it if it still belongs to us - if wass.GetActorUid() != actor.GetMetadata().GetUid() { - slog.WarnContext(ctx, "Worker already assigned to another Actor", slog.String("worker", workerName)) - return sandboxClass, nil + // Release only this actor's assignment; the worker may be hosting others, + // and they are unaffected by this one crashing. A worker that is no longer + // hosting it has already been released. + released, err := st.ReleaseActorFromWorker(ctx, workerName, actor.GetMetadata().GetUid()) + if err != nil { + return sandboxClass, nil, fmt.Errorf("while releasing worker: %w", err) } - - if _, err := st.UpdateWorker(ctx, workerName, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.Assignment = nil - return nil - }); err != nil { - return sandboxClass, fmt.Errorf("while releasing worker: %w", err) + if released == nil { + slog.WarnContext(ctx, "Worker is not hosting this Actor, skipping release", + slog.String("worker", workerName)) } - return sandboxClass, nil + return sandboxClass, released, nil } diff --git a/cmd/ateapi/internal/controlapi/crash_test.go b/cmd/ateapi/internal/controlapi/crash_test.go index de4a0cb4ad..a7308e4902 100644 --- a/cmd/ateapi/internal/controlapi/crash_test.go +++ b/cmd/ateapi/internal/controlapi/crash_test.go @@ -66,23 +66,23 @@ func seedWorker(t *testing.T, ctx context.Context, st store.Interface, actorRef WorkerPodUid: "uid", Status: &ateapipb.WorkerStatus{}, } - if actorRef != (resources.ActorRef{}) { - actor, err := st.GetActor(ctx, actorRef) - if err != nil { - worker.Status.Assignment = &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: actorRef.Atespace, Name: actorRef.Name}, - ActorUid: "synthetic-" + actorRef.Name, - } - } else { - worker.Status.Assignment = &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: actor.GetMetadata().GetAtespace(), Name: actor.GetMetadata().GetName()}, - ActorUid: actor.GetMetadata().GetUid(), - } - } - } if _, err := st.CreateWorker(ctx, worker); err != nil { t.Fatalf("seed worker: %v", err) } + if actorRef == (resources.ActorRef{}) { + return + } + assignment := &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorUid: "synthetic-" + actorRef.Name, + } + if actor, err := st.GetActor(ctx, actorRef); err == nil { + assignment = &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: actor.GetMetadata().GetAtespace(), Name: actor.GetMetadata().GetName()}, + ActorUid: actor.GetMetadata().GetUid(), + } + } + seedAssignment(t, st, "uid", assignment) } // seedUnboundActor stores a running actor whose worker-binding fields were @@ -150,12 +150,8 @@ func TestCrashActor(t *testing.T) { t.Fatalf("crashActor() = %v, want nil", err) } assertCrashed(t, ctx, st, actorRef) - worker, gerr := st.GetWorker(ctx, "uid") - if gerr != nil { - t.Fatalf("GetWorker() = %v, want nil", gerr) - } - if worker.GetStatus().GetAssignment() != nil { - t.Errorf("worker assignment = %v, want nil", worker.GetStatus().GetAssignment()) + if got := firstAssignment(t, st, "uid"); got != nil { + t.Errorf("worker assignment = %v, want none", got) } }, }, @@ -170,14 +166,14 @@ func TestCrashActor(t *testing.T) { t.Fatalf("crashActor() = %v, want nil", err) } assertCrashed(t, ctx, st, actorRef) - worker, gerr := st.GetWorker(ctx, "uid") - if gerr != nil { - t.Fatalf("GetWorker() = %v, want nil", gerr) + assigned := firstAssignment(t, st, "uid") + if assigned == nil { + t.Fatal("worker assignment = nil, want the other actor's, untouched") } - if got := worker.GetStatus().GetAssignment().GetActor().GetName(); got != "actor-2" { + if got := assigned.GetActor().GetName(); got != "actor-2" { t.Errorf("worker assigned actor name = %q, want %q", got, "actor-2") } - if got := worker.GetStatus().GetAssignment().GetActorUid(); got != "synthetic-actor-2" { + if got := assigned.GetActorUid(); got != "synthetic-actor-2" { t.Errorf("worker assigned actor uid = %q, want %q", got, "synthetic-actor-2") } }, @@ -193,30 +189,29 @@ func TestCrashActor(t *testing.T) { WorkerPool: "pool", WorkerPod: "pod", WorkerPodUid: "uid", - Status: &ateapipb.WorkerStatus{ - Assignment: &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: actorRef.Atespace, Name: actorRef.Name}, - ActorUid: "stale-incarnation-uid", - }, - }, + Status: &ateapipb.WorkerStatus{}, } if _, err := st.CreateWorker(ctx, worker); err != nil { t.Fatalf("CreateWorker: %v", err) } + seedAssignment(t, st, "uid", &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorUid: "stale-incarnation-uid", + }) }, check: func(t *testing.T, ctx context.Context, st store.Interface, err error) { if err != nil { t.Fatalf("crashActor() = %v, want nil", err) } assertCrashed(t, ctx, st, actorRef) - worker, gerr := st.GetWorker(ctx, "uid") - if gerr != nil { - t.Fatalf("GetWorker() = %v, want nil", gerr) + assigned := firstAssignment(t, st, "uid") + if assigned == nil { + t.Fatal("worker assignment = nil, want the stale incarnation's, untouched") } - if got := worker.GetStatus().GetAssignment().GetActor().GetName(); got != actorRef.Name { + if got := assigned.GetActor().GetName(); got != actorRef.Name { t.Errorf("worker assigned actor name = %q, want %q", got, actorRef.Name) } - if got := worker.GetStatus().GetAssignment().GetActorUid(); got != "stale-incarnation-uid" { + if got := assigned.GetActorUid(); got != "stale-incarnation-uid" { t.Errorf("worker assigned actor uid = %q, want %q", got, "stale-incarnation-uid") } }, @@ -236,11 +231,7 @@ func TestCrashActor(t *testing.T) { // Without a binding the worker cannot be looked up, so its // assignment must be left untouched even though it names // the crashed actor. - worker, gerr := st.GetWorker(ctx, "uid") - if gerr != nil { - t.Fatalf("GetWorker() = %v, want nil", gerr) - } - if worker.GetStatus().GetAssignment() == nil { + if firstAssignment(t, st, "uid") == nil { t.Error("worker assignment = nil, want untouched") } }, @@ -426,11 +417,7 @@ func TestCrashActor_Metrics(t *testing.T) { WorkerPod: "pod-1", WorkerPodUid: "pod-uid-1", SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{ - Assignment: &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: actorRef.Atespace, Name: actorRef.Name}, - }, - }, + Status: &ateapipb.WorkerStatus{}, } if _, err := st.CreateWorker(ctx, worker); err != nil { t.Fatalf("CreateWorker: %v", err) @@ -531,14 +518,14 @@ func assertNoCrashMetricDatapoint(t *testing.T, reader *sdkmetric.ManualReader) } } -// failingUpdateWorkerStore wraps a store and fails every UpdateWorker call, -// simulating a transient state-store error while releasing a worker. -type failingUpdateWorkerStore struct { +// failingReleaseStore wraps a store and fails every release, simulating a +// transient state-store error while releasing a worker. +type failingReleaseStore struct { store.Interface err error } -func (f failingUpdateWorkerStore) UpdateWorker(context.Context, string, store.Precondition, func(*ateapipb.Worker) error) (*ateapipb.Worker, error) { +func (f failingReleaseStore) ReleaseActorFromWorker(context.Context, string, string) (*ateapipb.Worker, error) { return nil, f.err } @@ -557,7 +544,7 @@ func TestCrashActorReleaseFailureLeavesWorkerReclaimable(t *testing.T) { seedWorker(t, ctx, st, actorRef) releaseErr := errors.New("state store unavailable") - err := crashActor(ctx, failingUpdateWorkerStore{Interface: st, err: releaseErr}, actorRef, ateattr.OperationUnknown, ateattr.ReasonUnknown) + err := crashActor(ctx, failingReleaseStore{Interface: st, err: releaseErr}, actorRef, ateattr.OperationUnknown, ateattr.ReasonUnknown) if err == nil { t.Fatal("crashActor() = nil, want error") @@ -581,11 +568,7 @@ func TestCrashActorReleaseFailureLeavesWorkerReclaimable(t *testing.T) { // The worker must still be assigned to the actor (the failed release did not // persist): it is not leaked, and a retry will reclaim it. - worker, werr := st.GetWorker(ctx, "uid") - if werr != nil { - t.Fatalf("GetWorker() = %v, want nil", werr) - } - if worker.GetStatus().GetAssignment() == nil { + if firstAssignment(t, st, "uid") == nil { t.Error("worker assignment = nil, want still assigned (release failed, must remain retriable)") } } diff --git a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go index bf04053056..585ad7f028 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/actor_test.go @@ -16,6 +16,7 @@ package functionaltest import ( "context" + "errors" "fmt" "strings" "sync" @@ -1793,18 +1794,15 @@ func TestResumeActor(t *testing.T) { SandboxClass: "gvisor", Labels: map[string]string{poolLabelKey: ns}, Status: &ateapipb.WorkerStatus{ - Assignment: &ateapipb.ActorAssignment{ - ActorTemplateRef: &ateapipb.ObjectRef{ - Atespace: testAtespace, - Name: "tmpl1", - }, - Actor: &ateapipb.ObjectRef{ - Name: name, - Atespace: testAtespace, - }, - ActorUid: getResp.GetMetadata().GetUid(), - }, State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, + Allocation: &ateapipb.WorkerAllocation{ + // Only the ceiling the worker reported. + Capacity: &ateapipb.WorkerResources{Actors: 1}, + // All a listing reports of the assignments. The actor declares + // no compute limits, so it registers as one actor and nothing + // else. + Allocated: &ateapipb.WorkerResources{Actors: 1}, + }, }, } @@ -2464,20 +2462,12 @@ func TestResumeActor_ReleasesStaleWorkerWhenPoolBecomesIneligible(t *testing.T) } switch w.GetWorkerPool() { case "pool-a": - if wass := w.GetStatus().GetAssignment(); wass != nil { - got := "" - if wass.Actor != nil { - got = wass.Actor.Name - } - t.Errorf("expected worker-a (now-ineligible pool-a) to be released, got actor name=%q", got) + if n := w.GetStatus().GetAllocation().GetAllocated().GetActors(); n != 0 { + t.Errorf("expected worker-a (now-ineligible pool-a) to be released, still holds %d actors", n) } case "pool-b": - if wass := w.GetStatus().GetAssignment(); wass != nil { - got := "" - if wass.Actor != nil { - got = wass.Actor.Name - } - t.Errorf("expected worker-b to stay free (actor crashed, not migrated), got actor name=%q", got) + if n := w.GetStatus().GetAllocation().GetAllocated().GetActors(); n != 0 { + t.Errorf("expected worker-b to stay free (actor crashed, not migrated), holds %d actors", n) } } } @@ -2588,8 +2578,8 @@ func TestResumeActor_CrashesIfAssignedWorkerIsDraining(t *testing.T) { continue } if w.GetWorkerPod() == assignedPod { - if w.GetStatus().GetAssignment() != nil { - t.Errorf("expected draining worker %q to be released, still assigned to %q", assignedPod, w.GetStatus().GetAssignment().GetActor().GetName()) + if n := w.GetStatus().GetAllocation().GetAllocated().GetActors(); n != 0 { + t.Errorf("expected draining worker %q to be released, still holds %d actors", assignedPod, n) } } } @@ -3230,3 +3220,54 @@ func TestCreateActor_RejectsUnknownRequestFields(t *testing.T) { _, err := tc.client.CreateActor(context.Background(), req) assertGrpcError(t, err, codes.InvalidArgument, "request: Invalid value: unknown field with protobuf tag 9999") } + +// The assignment commits before the Actor is updated to point at it, so a +// crash in between leaves a row no Actor references. Deleting the Actor has to +// release it anyway: nothing else ever would, and its share of the Worker's +// capacity would stay booked until the Worker itself went away. +func TestDeleteActor_ReleasesAnAssignmentTheActorDoesNotReference(t *testing.T) { + ns := namespaceForTest("ns-delete-orphan") + tc := setupTest(t, ns) + defer tc.cleanup() + + createTemplate(t, tc, ns) + createWorkerPool(t, tc, ns, "pool-1", nil) + podUID := createWorkerPod(t, tc, ns, "worker-1", "node-1", "pool-1") + + ctx := context.Background() + actor, err := tc.client.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: "orphaned"}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}) + if err != nil { + t.Fatalf("CreateActor failed: %v", err) + } + actorUID := actor.GetMetadata().GetUid() + + // Bind straight through the store, leaving the Actor's backlink unset: + // exactly the state a crash between the two writes leaves behind. + if err := tc.persistence.BindActorToWorker(ctx, podUID, &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "orphaned"}, + ActorUid: actorUID, + ActorTemplateRef: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }, nil); err != nil { + t.Fatalf("BindActorToWorker failed: %v", err) + } + + if _, err := tc.client.DeleteActor(ctx, &ateapipb.DeleteActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "orphaned"}, + }); err != nil { + t.Fatalf("DeleteActor failed: %v", err) + } + + if _, err := tc.persistence.GetWorkerAssignment(ctx, podUID, actorUID); !errors.Is(err, store.ErrNotFound) { + t.Errorf("the orphaned assignment survived DeleteActor: %v", err) + } + worker, err := tc.persistence.GetWorker(ctx, podUID) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + if got := worker.GetStatus().GetAllocation().GetAllocated().GetActors(); got != 0 { + t.Errorf("worker still books %d actors after the Actor was deleted, want 0", got) + } +} diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 9a8f44bfa9..2349d9b159 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -419,7 +419,15 @@ func createWorkerPool(t *testing.T, tc *testContext, ns string, name string, lab t.Fatalf("failed to create WorkerPool: %v", err) } - err = wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { + waitForWorkerPoolInInformer(t, tc, ns, name) +} + +// waitForWorkerPoolInInformer blocks until the pool the test just created is +// visible to the scheduler, which reads it through an informer rather than the +// API. +func waitForWorkerPoolInInformer(t *testing.T, tc *testContext, ns, name string) { + t.Helper() + err := wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { _, err := tc.workerPoolLister.WorkerPools(ns).Get(name) return err == nil, nil }) @@ -518,10 +526,15 @@ func createWorkerPod(t *testing.T, tc *testContext, ns string, name string, node NodeName: nodeName, SandboxClass: string(pool.Spec.SandboxClass), Labels: pool.GetLabels(), + // Capacity is not settable here: a Worker gets it from its own + // ateom's report, which the reportWorkerCapacity below stands in + // for. These pods declare no limits, so only the actor ceiling is + // reported; see setWorkerActorCapacity for tests needing more. }, }); err != nil { t.Fatalf("failed to register worker: %v", err) } + reportWorkerCapacity(t, tc, string(createdPod.UID), 1) // Wait for the worker to appear in worker cache. err = wait.PollUntilContextTimeout(context.Background(), 10*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { @@ -552,7 +565,10 @@ func waitForWorkerAvailable(t *testing.T, tc *testContext, workerName string) { if err != nil { return false, nil } - return worker.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_ACTIVE && worker.GetStatus().GetAssignment() == nil, nil + // Hosting nothing is what "available" means, and the allocation total + // is how a cached Worker reports it: it does not carry the records. + return worker.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_ACTIVE && + worker.GetStatus().GetAllocation().GetAllocated().GetActors() == 0, nil }) if err != nil { t.Fatalf("failed to wait for worker %s to become available: %v", workerName, err) diff --git a/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go b/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go new file mode 100644 index 0000000000..62e8bd71f0 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/functionaltest/worker_contention_test.go @@ -0,0 +1,179 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package functionaltest + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// TestResumeActor_ConcurrentOntoOneWorker is the regression test for a refusal +// that only concurrency produces, and that a worker hosting one actor could +// never have shown. +// +// Claiming a worker rewrites that worker's whole record, so actors activating +// onto the SAME worker are N writers compare-and-swapping one row. All but one +// lose each round. That is fine and expected -- the loser re-reads and tries +// again -- but only if the retry budget is sized for how many writers there +// are. It was five steps from 10ms, and on a real cluster that rejected 21% of +// activations at 12 in flight and 67% at 24, every one of them failing at +// ~235ms with "timed out waiting for the condition". +// +// Nothing sequential catches it: the same activations one at a time all +// succeed, which is exactly what the pre-existing resume tests do. +func TestResumeActor_ConcurrentOntoOneWorker(t *testing.T) { + const actors = 128 + + ns := namespaceForTest("ns-resume-contention") + tc := setupTest(t, ns) + defer tc.cleanup() + + // createTemplate makes pool1 for us; widen it so one worker has room for all + // of them. The point is that they contend, not that they are turned away for + // lack of capacity. + createTemplate(t, tc, ns) + createWorkerPod(t, tc, ns, "worker-1", "node1", "pool1") + setWorkerActorCapacity(t, tc, "pool1", actors) + + for i := range actors { + if _, err := tc.client.CreateActor(context.Background(), &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: testAtespace, Name: fmt.Sprintf("id%d", i)}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "tmpl1"}, + }}); err != nil { + t.Fatalf("CreateActor %d: %v", i, err) + } + } + + // Resume them all at once. Starting the goroutines is not enough to make + // them race -- release them together so they arrive at the claim inside the + // same window. + var start sync.WaitGroup + start.Add(1) + var wg sync.WaitGroup + errs := make([]error, actors) + for i := range actors { + wg.Go(func() { + start.Wait() + _, errs[i] = tc.client.ResumeActor(context.Background(), &ateapipb.ResumeActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: fmt.Sprintf("id%d", i)}, + }) + }) + } + start.Done() + wg.Wait() + + var failed int + for i, err := range errs { + if err != nil { + failed++ + if failed <= 3 { + t.Errorf("ResumeActor id%d: %v", i, err) + } + } + } + if failed > 0 { + t.Fatalf("%d of %d concurrent activations onto one worker were refused; "+ + "they contend on the worker record and the retry budget has to absorb that", failed, actors) + } + + // And every one of them is really on the worker: a claim that was lost but + // reported as won would show up here rather than as an error above. Found by + // pod rather than by taking the only worker in the list -- the list is + // process-wide, so a -count>1 run sees the workers earlier iterations made. + listed, err := tc.client.ListWorkers(context.Background(), &ateapipb.ListWorkersRequest{}) + if err != nil { + t.Fatalf("ListWorkers: %v", err) + } + var worker *ateapipb.Worker + for _, candidate := range listed.GetWorkers() { + if candidate.GetWorkerNamespace() == ns && candidate.GetWorkerPod() == "worker-1" { + worker = candidate + break + } + } + if worker == nil { + t.Fatalf("worker %s/worker-1 not in the list of %d", ns, len(listed.GetWorkers())) + } + // A listing reports occupancy through the allocation total; the assignments + // themselves are their own records. + if got := int(worker.GetStatus().GetAllocation().GetAllocated().GetActors()); got != actors { + t.Errorf("worker allocation counts %d actors, want %d", got, actors) + } + page, err := tc.persistence.ListWorkerAssignments(context.Background(), worker.GetMetadata().GetName(), store.ListOptions{}) + if err != nil { + t.Fatalf("ListWorkerAssignments: %v", err) + } + if len(page.Items) != actors { + t.Errorf("worker holds %d assignments, want %d", len(page.Items), actors) + } +} + +// setWorkerActorCapacity raises the actors ceiling on every Worker in the pool. +// +// The ceiling is the Worker's, reported by its ateom, so a test that wants more +// than the unset default of one writes it where the reporter would. Waits for +// the scheduler's cache to see it, since placement reads that and not the store. +func setWorkerActorCapacity(t *testing.T, tc *testContext, pool string, actors int32) { + t.Helper() + page, err := tc.persistence.ListWorkers(context.Background(), store.ListOptions{PageSize: 1000}) + if err != nil { + t.Fatalf("listing workers: %v", err) + } + var reported int + for _, w := range page.Items { + if w.GetWorkerPool() != pool { + continue + } + reportWorkerCapacity(t, tc, w.GetMetadata().GetName(), actors) + reported++ + } + if reported == 0 { + t.Fatalf("no workers in pool %q to give capacity", pool) + } +} + +// reportWorkerCapacity stands in for the ateom's capacity report, which atelet +// forwards to WorkerService in a real cluster. It waits for the worker cache, +// which placement reads, to catch up. +func reportWorkerCapacity(t *testing.T, tc *testContext, name string, actors int32) { + t.Helper() + ctx := context.Background() + worker, err := tc.persistence.GetWorker(ctx, name) + if err != nil { + t.Fatalf("getting worker %s: %v", name, err) + } + if _, err := tc.persistence.UpdateWorker(ctx, name, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { + resources.Allocation(toUpdate).Capacity = &ateapipb.WorkerResources{Actors: actors} + return nil + }); err != nil { + t.Fatalf("setting capacity on worker %s: %v", name, err) + } + if err := wait.PollUntilContextTimeout(ctx, 10*time.Millisecond, 5*time.Second, true, + func(context.Context) (bool, error) { + got, err := tc.workerCache.Worker(name) + return err == nil && got.GetStatus().GetAllocation().GetCapacity().GetActors() == actors, nil + }); err != nil { + t.Fatalf("worker %s did not reach capacity.actors=%d in the cache: %v", name, actors, err) + } +} diff --git a/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go index f36cd14334..e68824425f 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/worker_test.go @@ -54,8 +54,9 @@ func TestListWorkers(t *testing.T) { want := []*ateapipb.Worker{ { Metadata: &ateapipb.ResourceMetadata{ - Name: podUID, - Version: 1, + Name: podUID, + // Two writes: the registration, then the capacity report. + Version: 2, }, WorkerNamespace: ns, WorkerPool: "pool1", @@ -65,9 +66,7 @@ func TestListWorkers(t *testing.T) { Ip: "127.0.0.1", SandboxClass: "gvisor", Labels: map[string]string{"foo": "bar"}, - Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - }, + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}}}, }, } diff --git a/cmd/ateapi/internal/controlapi/metrics.go b/cmd/ateapi/internal/controlapi/metrics.go index a080f83768..c40420902a 100644 --- a/cmd/ateapi/internal/controlapi/metrics.go +++ b/cmd/ateapi/internal/controlapi/metrics.go @@ -96,8 +96,10 @@ func RegisterWorkerCount(meter metric.Meter, workers func() ([]*ateapipb.Worker, } } for _, w := range ws { + // Occupancy comes from the allocation total rather than the + // assignment list, which a listed worker does not carry. state := ateattr.WorkerStateIdle - if w.GetStatus().GetAssignment() != nil { + if w.GetStatus().GetAllocation().GetAllocated().GetActors() > 0 { state = ateattr.WorkerStateAssigned } tally[key{w.GetWorkerNamespace(), w.GetWorkerPool(), state, w.GetSandboxClass()}]++ diff --git a/cmd/ateapi/internal/controlapi/metrics_test.go b/cmd/ateapi/internal/controlapi/metrics_test.go index a30f780775..591e96a9e7 100644 --- a/cmd/ateapi/internal/controlapi/metrics_test.go +++ b/cmd/ateapi/internal/controlapi/metrics_test.go @@ -72,9 +72,9 @@ func mustMetric(t *testing.T, reader *sdkmetric.ManualReader, name string) metri } func worker(namespace, pool, class string, assigned bool) *ateapipb.Worker { - w := &ateapipb.Worker{WorkerNamespace: namespace, WorkerPool: pool, SandboxClass: class, Status: &ateapipb.WorkerStatus{}} + w := &ateapipb.Worker{WorkerNamespace: namespace, WorkerPool: pool, SandboxClass: class, Status: &ateapipb.WorkerStatus{Allocation: &ateapipb.WorkerAllocation{}}} if assigned { - w.Status.Assignment = &ateapipb.ActorAssignment{} + w.Status.Allocation.Allocated = &ateapipb.WorkerResources{Actors: 1} } return w } diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index 842ac7dc81..8db83acccf 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -114,6 +114,7 @@ type serviceStore interface { DeleteActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef) (*ateapipb.ActorTemplate, error) ListWorkers(ctx context.Context, opts store.ListOptions) (store.ListResponse[*ateapipb.Worker], error) GetWorker(ctx context.Context, name string) (*ateapipb.Worker, error) + ListWorkerAssignments(ctx context.Context, workerName string, opts store.ListOptions) (store.ListResponse[*ateapipb.ActorAssignment], error) CreateWorker(ctx context.Context, worker *ateapipb.Worker) (*ateapipb.Worker, error) UpdateWorker(ctx context.Context, name string, precondition store.Precondition, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) AcquireLease(ctx context.Context, key string) (*store.Lease, error) diff --git a/cmd/ateapi/internal/controlapi/worker.go b/cmd/ateapi/internal/controlapi/worker.go index d130a8a04d..eb6db48c37 100644 --- a/cmd/ateapi/internal/controlapi/worker.go +++ b/cmd/ateapi/internal/controlapi/worker.go @@ -30,6 +30,41 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" ) +// ListWorkerActorAssignments lists the Actors a Worker hosts. The assignments are a +// subresource rather than a field on Worker, so this is the only way to read +// them and neither GetWorker nor ListWorkers grows with occupancy. +func (s *RPCService) ListWorkerActorAssignments(ctx context.Context, req *ateapipb.ListWorkerActorAssignmentsRequest) (*ateapipb.ListWorkerActorAssignmentsResponse, error) { + if errs := validateListWorkerActorAssignmentsRequest(ctx, req); len(errs) > 0 { + return nil, toGRPCStatusError(errs) + } + name := req.GetWorker().GetName() + + // The Worker is read first so a listing against one that does not exist is + // NOT_FOUND rather than an empty page, which a caller cannot tell from a + // Worker hosting nothing. + if _, err := s.impl.GetWorker(ctx, name); err != nil { + if errors.Is(err, store.ErrNotFound) { + return nil, status.Errorf(codes.NotFound, "Worker %s not found", name) + } + return nil, fmt.Errorf("while fetching worker %s: %w", name, err) + } + + page, err := s.impl.ListWorkerAssignments(ctx, name, + store.ListOptions{PageSize: effectivePageSize(req.GetPageSize()), PageToken: req.GetPageToken()}) + if err != nil { + return nil, mapListError(fmt.Errorf("while listing the assignments of worker %s: %w", name, err)) + } + return &ateapipb.ListWorkerActorAssignmentsResponse{ + ActorAssignments: page.Items, + NextPageToken: page.NextPageToken, + }, nil +} + +func validateListWorkerActorAssignmentsRequest(ctx context.Context, req *ateapipb.ListWorkerActorAssignmentsRequest) field.ErrorList { + op := operation.Operation{Type: operation.Create} + return Validate_ListWorkerActorAssignmentsRequest(ctx, op, nil, req, nil) +} + func (s *RPCService) ListWorkers(ctx context.Context, req *ateapipb.ListWorkersRequest) (*ateapipb.ListWorkersResponse, error) { if errs := validateListWorkersRequest(ctx, req); len(errs) > 0 { return nil, toGRPCStatusError(errs) @@ -71,6 +106,10 @@ func (s *RPCService) GetWorker(ctx context.Context, req *ateapipb.GetWorkerReque return worker, nil } +// GetWorker returns the Worker with the Actors it hosts, read separately since +// the assignments are their own records. The one read that pays O(assignments). +// A failed read is an error, not a Worker reported as hosting nothing: the +// caller cannot tell those apart. func (s *ServiceImpl) GetWorker(ctx context.Context, name string) (*ateapipb.Worker, error) { return s.store.GetWorker(ctx, name) } @@ -106,6 +145,11 @@ func (s *ServiceImpl) CreateWorker(ctx context.Context, inWorker *ateapipb.Worke outWorker := proto.CloneOf(inWorker) outWorker.Status = &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE} + // Capacity is left unset: a Worker holds nothing until its own ateom says + // what it has, through WorkerService.SetWorkerCapacity. Nothing is placed + // on it in the meantime, which is the point -- the alternative is guessing + // on the Worker's behalf and placing against the guess. + // Verify that the result is properly valid before storing it. if errs := validateWorkerUpdate(ctx, field.NewPath("worker"), outWorker, inWorker, true); len(errs) > 0 { return nil, toGRPCInternalError(errs) @@ -196,6 +240,29 @@ func validateUpdateWorkerRequest(ctx context.Context, req *ateapipb.UpdateWorker return Validate_UpdateWorkerRequest(ctx, op, nil, req, nil) } +// The assignment operations are pass-throughs: an assignment is its own record, +// so binding and releasing are single store calls rather than a read-modify-write +// of the Worker. +func (s *ServiceImpl) BindActorToWorker(ctx context.Context, workerName string, assignment *ateapipb.ActorAssignment, admit func(*ateapipb.Worker) error) error { + return s.store.BindActorToWorker(ctx, workerName, assignment, admit) +} + +func (s *ServiceImpl) ReleaseActorFromWorker(ctx context.Context, workerName string, actorUID string) (*ateapipb.Worker, error) { + return s.store.ReleaseActorFromWorker(ctx, workerName, actorUID) +} + +func (s *ServiceImpl) GetWorkerAssignment(ctx context.Context, workerName, actorUID string) (*ateapipb.ActorAssignment, error) { + return s.store.GetWorkerAssignment(ctx, workerName, actorUID) +} + +func (s *ServiceImpl) ListWorkerAssignments(ctx context.Context, workerName string, opts store.ListOptions) (store.ListResponse[*ateapipb.ActorAssignment], error) { + return s.store.ListWorkerAssignments(ctx, workerName, opts) +} + +func (s *ServiceImpl) FindWorkerHostingActor(ctx context.Context, actorUID string) (string, error) { + return s.store.FindWorkerHostingActor(ctx, actorUID) +} + func (s *RPCService) DeleteWorker(ctx context.Context, req *ateapipb.DeleteWorkerRequest) (*ateapipb.Worker, error) { if errs := validateDeleteWorkerRequest(ctx, req); len(errs) > 0 { return nil, toGRPCStatusError(errs) @@ -244,9 +311,8 @@ func (s *RPCService) DrainWorker(ctx context.Context, req *ateapipb.DrainWorkerR return &workerUnchanged{worker: proto.Clone(toUpdate).(*ateapipb.Worker)} } toUpdate.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING - // status.assignment is deliberately left alone: a draining Worker keeps - // hosting the Actor bound to it until something releases it. Draining - // only stops the scheduler routing new Actors here. + // The assignments are left alone: a draining Worker keeps its Actors + // until something releases them. Draining only stops new placements. return nil }) } diff --git a/cmd/ateapi/internal/controlapi/worker_assignments.go b/cmd/ateapi/internal/controlapi/worker_assignments.go new file mode 100644 index 0000000000..cb44bc0202 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/worker_assignments.go @@ -0,0 +1,42 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "errors" + "fmt" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +type workerAssignmentReader interface { + GetWorkerAssignment(ctx context.Context, workerName, actorUID string) (*ateapipb.ActorAssignment, error) +} + +// workerHostsActor reports whether a Worker holds an assignment for actorUID. +// It asks the store: the Worker record does not carry its assignments, and the +// watch-fed cache cannot see a binding committed moments ago. +func workerHostsActor(ctx context.Context, st workerAssignmentReader, workerName, actorUID string) (bool, error) { + _, err := st.GetWorkerAssignment(ctx, workerName, actorUID) + if errors.Is(err, store.ErrNotFound) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("while checking whether worker %q hosts actor %q: %w", workerName, actorUID, err) + } + return true, nil +} diff --git a/cmd/ateapi/internal/controlapi/worker_test.go b/cmd/ateapi/internal/controlapi/worker_test.go index 30c202084f..80974f79fc 100644 --- a/cmd/ateapi/internal/controlapi/worker_test.go +++ b/cmd/ateapi/internal/controlapi/worker_test.go @@ -17,11 +17,13 @@ package controlapi import ( "context" "fmt" + "sort" "strings" "testing" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "github.com/google/go-cmp/cmp" "google.golang.org/grpc/codes" @@ -49,7 +51,6 @@ func validWorker(name string, mods ...func(*ateapipb.Worker)) *ateapipb.Worker { NodeName: "node-1", Ip: "10.1.2.3", SandboxClass: "gvisor", - Capacity: &ateapipb.WorkerCapacity{CpuMilli: 2000, MemoryBytes: 4 << 30}, } for _, m := range mods { m(w) @@ -87,12 +88,16 @@ func newAPIAssignment(actorUID string) *ateapipb.ActorAssignment { // newWorkerAPIService returns a service backed by a real store, which is what // makes the compare-and-set assertions below meaningful — a fake would decide // the outcome the test is trying to observe. +// impl is a real ServiceImpl rather than the store itself, as main wires it: +// the service layer is where a read composes the Worker with records kept +// outside it, so a test that hands RPCService the bare store silently skips +// that and reports whatever the store row happens to hold. func newWorkerAPIService(t *testing.T) (*RPCService, store.Interface) { t.Helper() persistence, cleanup := storetest.SetupTestStore(t) t.Cleanup(cleanup) impl := newServiceImpl(persistence, nil) - return &RPCService{impl: impl, workerWorkflow: NewWorkerWorkflow(impl)}, persistence + return &RPCService{impl: impl, workerWorkflow: NewWorkerWorkflow(persistence)}, persistence } // seedAPIWorker registers a worker directly through the store and returns it as @@ -114,16 +119,12 @@ func seedAPIWorker(t *testing.T, ctx context.Context, persistence store.Interfac // in-process, through the store. There is no AssignWorker RPC to go through. func assignAPIWorker(t *testing.T, ctx context.Context, persistence store.Interface, name, actorUID string) *ateapipb.Worker { t.Helper() - observed, err := persistence.GetWorker(ctx, name) - if err != nil { - t.Fatalf("getting worker %s to assign: %v", name, err) + if err := persistence.BindActorToWorker(ctx, name, newAPIAssignment(actorUID), nil); err != nil { + t.Fatalf("assigning worker %s: %v", name, err) } - assigned, err := persistence.UpdateWorker(ctx, name, store.PreconditionFrom(observed), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.Assignment = newAPIAssignment(actorUID) - return nil - }) + assigned, err := persistence.GetWorker(ctx, name) if err != nil { - t.Fatalf("assigning worker %s: %v", name, err) + t.Fatalf("re-reading worker %s after assigning: %v", name, err) } return assigned } @@ -145,50 +146,71 @@ func updateFrom(observed *ateapipb.Worker, mutate func(*ateapipb.Worker)) *ateap return worker } -func TestValidateListWorkersRequest(t *testing.T) { - tests := []struct { - name string - req *ateapipb.ListWorkersRequest - want field.ErrorList - }{{ - "valid, no page_size", - &ateapipb.ListWorkersRequest{}, - nil, - }, { - "valid, positive page_size", - &ateapipb.ListWorkersRequest{PageSize: 10}, - nil, - }, { - "negative page_size", - &ateapipb.ListWorkersRequest{PageSize: -1}, - field.ErrorList{field.Invalid(field.NewPath("page_size"), int32(-1), "").WithOrigin("minimum")}, - }, { - "valid page_token", - &ateapipb.ListWorkersRequest{PageToken: strings.Repeat("x", 256)}, - nil, - }, { - "too-large page_token", - &ateapipb.ListWorkersRequest{PageToken: strings.Repeat("x", 257)}, - field.ErrorList{field.TooLongCharacters(field.NewPath("page_token"), "", 256).WithOrigin("maxLength")}, - }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assertValidateErr(t, validateListWorkersRequest(context.Background(), tt.req), tt.want) - }) +// The assignments are a subresource, so this RPC is the only way to read them. +// It also pins the identity the store gives each one. +func TestListWorkerAssignments(t *testing.T) { + ctx := context.Background() + svc, persistence := newWorkerAPIService(t) + seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) + assignAPIWorker(t, ctx, persistence, apiWorkerName, "actor-uid-1") + assignAPIWorker(t, ctx, persistence, apiWorkerName, "actor-uid-2") + + page, err := svc.ListWorkerActorAssignments(ctx, &ateapipb.ListWorkerActorAssignmentsRequest{Worker: workerRef(apiWorkerName)}) + if err != nil { + t.Fatalf("ListWorkerActorAssignments() failed: %v", err) + } + var uids []string + for _, a := range page.GetActorAssignments() { + uids = append(uids, a.GetActorUid()) + if got := a.GetMetadata().GetName(); got != a.GetActorUid() { + t.Errorf("assignment name = %q, want the Actor uid %q", got, a.GetActorUid()) + } + if got := a.GetMetadata().GetAtespace(); got != "" { + t.Errorf("assignment atespace = %q, want empty: Workers are global-scoped", got) + } + if a.GetMetadata().GetUid() == "" { + t.Error("assignment uid is unset, want the one the store generated") + } + } + sort.Strings(uids) + if diff := cmp.Diff([]string{"actor-uid-1", "actor-uid-2"}, uids); diff != "" { + t.Errorf("assignments mismatch (-want +got):\n%s", diff) + } +} + +// An absent Worker is NOT_FOUND rather than an empty page, which a caller +// cannot tell from a Worker hosting nothing. +func TestListWorkerAssignments_AbsentWorker(t *testing.T) { + ctx := context.Background() + svc, _ := newWorkerAPIService(t) + + _, err := svc.ListWorkerActorAssignments(ctx, &ateapipb.ListWorkerActorAssignmentsRequest{ + Worker: workerRef("3b9f1e77-2c4d-4a80-91be-6d5c8f0a7e21"), + }) + if got := status.Code(err); got != codes.NotFound { + t.Fatalf("code = %v (err %v), want %v", got, err, codes.NotFound) } } -func TestGetWorker(t *testing.T) { +// Capacity is reported by the Worker, so it lives in status and a client +// cannot move it. An update that tries is ignored rather than refused, as it is +// for anything else a request carries in status. +func TestUpdateWorker_CannotChangeCapacity(t *testing.T) { ctx := context.Background() svc, persistence := newWorkerAPIService(t) - want := seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) + seeded := seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) + before := seeded.GetStatus().GetAllocation().GetCapacity() - got, err := svc.GetWorker(ctx, &ateapipb.GetWorkerRequest{Worker: workerRef(apiWorkerName)}) + got, err := svc.UpdateWorker(ctx, &ateapipb.UpdateWorkerRequest{ + Worker: updateFrom(seeded, func(w *ateapipb.Worker) { + resources.Allocation(w).Capacity = &ateapipb.WorkerResources{Actors: 4094} + }), + }) if err != nil { - t.Fatalf("GetWorker() failed: %v", err) + t.Fatalf("UpdateWorker() failed: %v", err) } - if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { - t.Errorf("GetWorker() mismatch (-want +got):\n%s", diff) + if diff := cmp.Diff(before, got.GetStatus().GetAllocation().GetCapacity(), protocmp.Transform()); diff != "" { + t.Errorf("a client update moved capacity (-want +got):\n%s", diff) } } @@ -255,10 +277,7 @@ func TestCreateWorker_IgnoresRequestStatus(t *testing.T) { svc, _ := newWorkerAPIService(t) in := validWorker(apiWorkerName) - in.Status = &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_DRAINING, - Assignment: newAPIAssignment("actor-uid-1"), - } + in.Status = &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_DRAINING, Allocation: &ateapipb.WorkerAllocation{Allocated: &ateapipb.WorkerResources{Actors: 9}}} got, err := svc.CreateWorker(ctx, &ateapipb.CreateWorkerRequest{Worker: in}) if err != nil { @@ -468,11 +487,13 @@ func TestUpdateWorker_Errors(t *testing.T) { {"ip changed", func(w *ateapipb.Worker) { w.Ip = "10.9.9.9" }, codes.InvalidArgument}, {"worker_pod changed", func(w *ateapipb.Worker) { w.WorkerPod = "worker-pod-2" }, codes.InvalidArgument}, {"node_name changed", func(w *ateapipb.Worker) { w.NodeName = "node-2" }, codes.InvalidArgument}, - {"capacity changed", func(w *ateapipb.Worker) { w.Capacity.CpuMilli = 4000 }, codes.InvalidArgument}, + // capacity is deliberately absent here: it may change (the pool's actor + // ceiling moves, a pod can be resized, a Worker may report its own). + // TestUpdateWorker_CapacityChanges covers that. + // // And immutable fields dropped, which a replacement update reads as a // request to clear them. Rejected rather than silently applied. {"ip omitted", func(w *ateapipb.Worker) { w.Ip = "" }, codes.InvalidArgument}, - {"capacity omitted", func(w *ateapipb.Worker) { w.Capacity = nil }, codes.InvalidArgument}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -522,7 +543,13 @@ func TestDeleteWorker(t *testing.T) { if err != nil { t.Fatalf("DeleteWorker() failed: %v", err) } - if diff := cmp.Diff(seeded, got, protocmp.Transform()); diff != "" { + // Delete drains before it sweeps, so it removes one revision past what was + // seeded. + want := proto.Clone(seeded).(*ateapipb.Worker) + want.Metadata.Version = seeded.GetMetadata().GetVersion() + 1 + want.Metadata.UpdateTime = got.GetMetadata().GetUpdateTime() + want.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { t.Errorf("DeleteWorker() returned something other than the worker it removed (-want +got):\n%s", diff) } if _, err := persistence.GetWorker(ctx, apiWorkerName); err == nil { @@ -543,6 +570,24 @@ func TestDeleteWorker_Absent(t *testing.T) { } } +// An assigned worker deletes like any other: the delete does not cascade, and +// an Actor pointing at a Worker that is gone is an expected steady state. +func TestDeleteWorker_AssignedWorkerDeletesAnyway(t *testing.T) { + ctx := context.Background() + svc, persistence := newWorkerAPIService(t) + seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) + assignAPIWorker(t, ctx, persistence, apiWorkerName, "actor-uid-1") + + got, err := svc.DeleteWorker(ctx, &ateapipb.DeleteWorkerRequest{Worker: workerRef(apiWorkerName)}) + if err != nil { + t.Fatalf("DeleteWorker() failed: %v", err) + } + // The record carries the total, not the assignments themselves. + if n := got.GetStatus().GetAllocation().GetAllocated().GetActors(); n != 1 { + t.Errorf("deleted worker hosted %d actors, want the 1 it was holding", n) + } +} + func TestDeleteWorker_Preconditions(t *testing.T) { ctx := context.Background() svc, persistence := newWorkerAPIService(t) @@ -625,8 +670,8 @@ func TestDrainWorker_KeepsAssignment(t *testing.T) { if err != nil { t.Fatalf("DrainWorker() failed: %v", err) } - if got.GetStatus().GetAssignment().GetActorUid() != "actor-uid-1" { - t.Errorf("assignment = %v, want it left in place", got.GetStatus().GetAssignment()) + if n := got.GetStatus().GetAllocation().GetAllocated().GetActors(); n != 1 { + t.Errorf("drained worker hosts %d actors, want the 1 left in place", n) } } @@ -778,20 +823,6 @@ func TestValidateCreateWorkerRequest(t *testing.T) { name: "invalid label value", req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.Labels = map[string]string{"tier": "not valid!"} })), want: field.ErrorList{field.Invalid(field.NewPath("worker", "labels").Key("tier"), "not valid!", "").WithOrigin("format=k8s-label-value")}, - }, { - name: "absent capacity is allowed", - req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.Capacity = nil })), - }, { - name: "valid capacity", - req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.Capacity = &ateapipb.WorkerCapacity{CpuMilli: 2000, MemoryBytes: 4 << 30} })), - }, { - name: "negative capacity.cpu_milli", - req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.Capacity = &ateapipb.WorkerCapacity{CpuMilli: -1, MemoryBytes: 4 << 30} })), - want: field.ErrorList{field.Invalid(field.NewPath("worker", "capacity", "cpu_milli"), nil, "").WithOrigin("minimum")}, - }, { - name: "negative capacity.memory_bytes", - req: validReq(validWorker(apiWorkerName, func(w *ateapipb.Worker) { w.Capacity = &ateapipb.WorkerCapacity{CpuMilli: 2000, MemoryBytes: -1} })), - want: field.ErrorList{field.Invalid(field.NewPath("worker", "capacity", "memory_bytes"), nil, "").WithOrigin("minimum")}, }, { name: "status needs a state", req: validReq(validWorker(apiWorkerName, withStatus(func(s *ateapipb.WorkerStatus) { s.State = 0 }))), @@ -804,45 +835,6 @@ func TestValidateCreateWorkerRequest(t *testing.T) { name: "status invalid state (too large)", req: validReq(validWorker(apiWorkerName, withStatus(func(s *ateapipb.WorkerStatus) { s.State = 99 }))), want: field.ErrorList{field.Invalid(field.NewPath("worker", "status", "state"), nil, "").WithOrigin("maximum")}, - }, { - name: "valid assignment, when carried, passes", - req: validReq(validWorker(apiWorkerName, withStatus(func(s *ateapipb.WorkerStatus) { - s.Assignment = newAPIAssignment(apiOtherWorkerName) - }))), - }, { - name: "assignment actor_uid must be a uuid", - req: validReq(validWorker(apiWorkerName, withStatus(func(s *ateapipb.WorkerStatus) { - s.Assignment = newAPIAssignment("not a uuid") - }))), - want: field.ErrorList{field.Invalid(field.NewPath("worker", "status", "assignment", "actor_uid"), nil, "").WithOrigin("format=k8s-uuid")}, - }, { - name: "assignment actor ref needs an atespace", - req: validReq(validWorker(apiWorkerName, withStatus(func(s *ateapipb.WorkerStatus) { - s.Assignment = newAPIAssignment(apiOtherWorkerName) - s.Assignment.Actor.Atespace = "" - }))), - want: field.ErrorList{field.Required(field.NewPath("worker", "status", "assignment", "actor", "atespace"), "")}, - }, { - name: "assignment template ref needs an atespace", - req: validReq(validWorker(apiWorkerName, withStatus(func(s *ateapipb.WorkerStatus) { - s.Assignment = newAPIAssignment(apiOtherWorkerName) - s.Assignment.ActorTemplateRef = &ateapipb.ObjectRef{Name: "tmpl"} - }))), - want: field.ErrorList{field.Required(field.NewPath("worker", "status", "assignment", "actor_template_ref", "atespace"), "")}, - }, { - name: "assignment needs a template ref", - req: validReq(validWorker(apiWorkerName, withStatus(func(s *ateapipb.WorkerStatus) { - s.Assignment = newAPIAssignment(apiOtherWorkerName) - s.Assignment.ActorTemplateRef = nil - }))), - want: field.ErrorList{field.Required(field.NewPath("worker", "status", "assignment", "actor_template_ref"), "")}, - }, { - name: "assignment template name must be a short name", - req: validReq(validWorker(apiWorkerName, withStatus(func(s *ateapipb.WorkerStatus) { - s.Assignment = newAPIAssignment(apiOtherWorkerName) - s.Assignment.ActorTemplateRef.Name = "TMPL_1" - }))), - want: field.ErrorList{field.Invalid(field.NewPath("worker", "status", "assignment", "actor_template_ref", "name"), nil, "").WithOrigin("format=k8s-short-name")}, }} for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -876,10 +868,8 @@ func TestServiceImplUpdateWorker_ImmutableFields(t *testing.T) { {"worker_pod_uid", "worker_pod_uid", func(w *ateapipb.Worker) { w.WorkerPodUid = apiOtherWorkerName }}, {"node_name", "node_name", func(w *ateapipb.Worker) { w.NodeName = "other-node" }}, {"ip", "ip", func(w *ateapipb.Worker) { w.Ip = "10.0.0.9" }}, - {"capacity_changed", "capacity", func(w *ateapipb.Worker) { w.Capacity.CpuMilli = 4000 }}, - // An update replaces the worker, so a caller that leaves capacity - // out is asking to clear it. That is a change like any other. - {"capacity_cleared", "capacity", func(w *ateapipb.Worker) { w.Capacity = nil }}, + // capacity is absent: it is reported into status, which a client + // cannot write. See TestUpdateWorker_CannotChangeCapacity. } { t.Run(tc.name, func(t *testing.T) { _, err := impl.UpdateWorker(ctx, apiWorkerName, store.PreconditionFrom(created), func(toUpdate *ateapipb.Worker) error { @@ -1061,38 +1051,30 @@ func TestCreateWorker_IgnoresRequestMetadataServerFields(t *testing.T) { } } -// TestServiceImplUpdateWorker_ValidatesAssignment pins that assignment writes -// — which reach the store through ServiceImpl, the way the resume workflow -// binds an Actor — are validated like any other worker update. -func TestServiceImplUpdateWorker_ValidatesAssignment(t *testing.T) { +// Every stored Worker carries an actor ceiling, so no reader has to know a +// default. A Worker that reports its own keeps it; one that does not is worth +// one Actor, which is what a Worker was before it could report. +func TestCreateWorker_HoldsNoCapacityUntilReported(t *testing.T) { ctx := context.Background() - persistence, cleanup := storetest.SetupTestStore(t) - t.Cleanup(cleanup) - impl := newServiceImpl(persistence, nil) - created := seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) + svc, _ := newWorkerAPIService(t) - // A malformed assignment must not land. - _, err := impl.UpdateWorker(ctx, apiWorkerName, store.PreconditionFrom(created), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.Assignment = newAPIAssignment("not-a-uuid") - return nil - }) - if got := status.Code(err); got != codes.InvalidArgument { - t.Fatalf("assigning a malformed uid returned %v (err %v), want %v", got, err, codes.InvalidArgument) + got, err := svc.CreateWorker(ctx, &ateapipb.CreateWorkerRequest{Worker: validWorker(apiWorkerName)}) + if err != nil { + t.Fatalf("CreateWorker() failed: %v", err) + } + if capacity := got.GetStatus().GetAllocation().GetCapacity(); capacity != nil { + t.Errorf("created worker capacity = %v, want none until its ateom reports", capacity) } - // A well-formed assignment lands, and releasing it lands too: assignment - // is optional, so clearing is not otherwise constrained. - assigned, err := impl.UpdateWorker(ctx, apiWorkerName, store.PreconditionFrom(created), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.Assignment = newAPIAssignment(apiOtherWorkerName) - return nil - }) + // Capacity is status, so a request cannot bring its own: a Worker only + // gets one by reporting it. + carried := validWorker("11111111-2222-3333-4444-555555555555") + carried.Status = &ateapipb.WorkerStatus{Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 4094}}} + got, err = svc.CreateWorker(ctx, &ateapipb.CreateWorkerRequest{Worker: carried}) if err != nil { - t.Fatalf("assigning a valid assignment failed: %v", err) + t.Fatalf("CreateWorker() carrying a capacity failed: %v", err) } - if _, err := impl.UpdateWorker(ctx, apiWorkerName, store.PreconditionFrom(assigned), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.Assignment = nil - return nil - }); err != nil { - t.Fatalf("releasing the assignment failed: %v", err) + if capacity := got.GetStatus().GetAllocation().GetCapacity(); capacity != nil { + t.Errorf("a request carrying a ceiling set capacity to %v, want none", capacity) } } diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index d4e41eea85..899968cd3f 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -114,6 +114,13 @@ type actorWorkflowStore interface { DeleteActor(ctx context.Context, actorRef resources.ActorRef) (*ateapipb.Actor, error) GetWorker(ctx context.Context, name string) (*ateapipb.Worker, error) UpdateWorker(ctx context.Context, name string, precondition store.Precondition, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) + BindActorToWorker(ctx context.Context, workerName string, assignment *ateapipb.ActorAssignment, admit func(*ateapipb.Worker) error) error + ReleaseActorFromWorker(ctx context.Context, workerName string, actorUID string) (*ateapipb.Worker, error) + GetWorkerAssignment(ctx context.Context, workerName, actorUID string) (*ateapipb.ActorAssignment, error) + FindWorkerHostingActor(ctx context.Context, actorUID string) (string, error) + // Read from the records rather than the Worker's status: only the service + // layer attaches assignments on read, and this workflow holds the store. + ListWorkerAssignments(ctx context.Context, workerName string, opts store.ListOptions) (store.ListResponse[*ateapipb.ActorAssignment], error) GetActorSnapshot(ctx context.Context, snapshotRef resources.ActorSnapshotRef) (*ateapipb.ActorSnapshot, error) CreateActorSnapshot(ctx context.Context, snapshot *ateapipb.ActorSnapshot) (*ateapipb.ActorSnapshot, error) GetActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef) (*ateapipb.ActorTemplate, error) @@ -138,7 +145,9 @@ func NewWorkerWorkflow(store workerWorkflowStore) *WorkerWorkflow { // WorkerWorkflow and nothing more. type workerWorkflowStore interface { GetWorker(ctx context.Context, name string) (*ateapipb.Worker, error) + UpdateWorker(ctx context.Context, name string, precondition store.Precondition, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) DeleteWorker(ctx context.Context, name string, pre store.DeletePreconditions) (*ateapipb.Worker, error) + ListWorkerAssignments(ctx context.Context, workerName string, opts store.ListOptions) (store.ListResponse[*ateapipb.ActorAssignment], error) GetActor(ctx context.Context, actorRef resources.ActorRef) (*ateapipb.Actor, error) UpdateActor(ctx context.Context, actorRef resources.ActorRef, precondition store.Precondition, mutate func(toUpdate *ateapipb.Actor) error) (*ateapipb.Actor, error) } diff --git a/cmd/ateapi/internal/controlapi/workflow_delete.go b/cmd/ateapi/internal/controlapi/workflow_delete.go index a96e40b63f..2205f5a766 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete.go @@ -127,8 +127,14 @@ func (w *ActorWorkflow) ensureAteletTerminated(ctx context.Context, actorRef res } return fmt.Errorf("while checking worker assignment: %w", err) } - wass := worker.GetStatus().GetAssignment() - if wass == nil || wass.GetActorUid() != actor.GetMetadata().GetUid() { + // Ask whether the worker still HOSTS this actor, not whether its one + // assignment happens to be this actor: a worker hosting several is the + // ordinary case, and the others are none of this delete's business. + hosted, err := workerHostsActor(ctx, w.store, worker.GetMetadata().GetName(), actor.GetMetadata().GetUid()) + if err != nil { + return err + } + if !hosted { slog.InfoContext(ctx, "worker is no longer assigned to this actor, skipping atelet terminate request", slog.String("worker", workerName), slog.Any("actor", actorRef)) @@ -213,13 +219,54 @@ func (w *ActorWorkflow) ensureVolumesDetachedForDelete(ctx context.Context, acto } // ensureWorkerReleased releases the worker assigned to the actor. +// releaseAssignmentWithoutBacklink releases an assignment the Actor does not +// reference, found by Actor UID. Absent is the ordinary case and not an error: +// most Actors reaching here really were released already. +func (w *ActorWorkflow) releaseAssignmentWithoutBacklink(ctx context.Context, actor *ateapipb.Actor) error { + actorUID := actor.GetMetadata().GetUid() + workerName, err := w.store.FindWorkerHostingActor(ctx, actorUID) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + markSkipped(ctx, "worker already released") + return nil + } + return fmt.Errorf("while looking for a worker still hosting actor %s: %w", actorUID, err) + } + + // Read only to learn whether the Worker is still there; the release itself + // is guarded by the assignment key. + if _, err := w.store.GetWorker(ctx, workerName); err != nil { + if errors.Is(err, store.ErrNotFound) { + markSkipped(ctx, "worker already released") + return nil + } + return fmt.Errorf("while getting worker %s to release: %w", workerName, err) + } + + slog.InfoContext(ctx, "Releasing an assignment the Actor does not reference", + slog.String("worker", workerName), slog.String("actor_uid", actorUID)) + _, err = w.store.ReleaseActorFromWorker(ctx, workerName, actorUID) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return nil + } + return fmt.Errorf("while releasing worker %s: %w", workerName, err) + } + return nil +} + func (w *ActorWorkflow) ensureWorkerReleased(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor) (updated *ateapipb.Actor, err error) { ctx, done := stepSpan(ctx, "ReleaseWorker") defer func() { err = done(err) }() if actor.GetStatus().GetWorkerAssignment() == nil { - markSkipped(ctx, "worker already released") - return actor, nil + // An Actor with no backlink may still be bound. The assignment commits + // before the Actor is updated to point at it, so a crash in between + // leaves a row nothing on the Actor names. Releasing by Actor UID is + // what recovers it; skipping would leave its share of the Worker's + // capacity booked until the Worker itself went away. The resume path + // recovers the same window through workerHoldingStaleClaim. + return actor, w.releaseAssignmentWithoutBacklink(ctx, actor) } latestActor, err := w.store.GetActor(ctx, actorRef) @@ -228,7 +275,8 @@ func (w *ActorWorkflow) ensureWorkerReleased(ctx context.Context, actorRef resou } if latestActor.GetStatus().GetWorkerAssignment() != nil { - if _, err := releaseWorker(ctx, w.store, latestActor); err != nil { + _, _, err := releaseWorker(ctx, w.store, latestActor) + if err != nil { return nil, err } diff --git a/cmd/ateapi/internal/controlapi/workflow_pause.go b/cmd/ateapi/internal/controlapi/workflow_pause.go index 07779c25f8..717cdecf23 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause.go @@ -235,21 +235,14 @@ func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef reso slog.Warn("Worker already gone during finalize pause, skipping release", "worker", assignment.GetWorkerPod()) } else { nodeName = worker.GetNodeName() - // Only free it if it still belongs to us - - if wass := worker.GetStatus().GetAssignment(); wass != nil { - if wass.GetActorUid() == latestActor.GetMetadata().GetUid() { - _, err := w.store.UpdateWorker(ctx, worker.GetMetadata().GetName(), store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.Assignment = nil - return nil - }) - if err != nil { - if errors.Is(err, store.ErrVersionConflict) { - return nil, status.Error(codes.Aborted, "concurrent update conflict, please retry") - } - return nil, err - } + // Drop just this actor's assignment; any other actors the worker + // hosts keep theirs. + _, err := w.store.ReleaseActorFromWorker(ctx, worker.GetMetadata().GetName(), latestActor.GetMetadata().GetUid()) + if err != nil { + if errors.Is(err, store.ErrVersionConflict) { + return nil, status.Error(codes.Aborted, "concurrent update conflict, please retry") } + return nil, err } } diff --git a/cmd/ateapi/internal/controlapi/workflow_pause_test.go b/cmd/ateapi/internal/controlapi/workflow_pause_test.go index e9b6e0ee33..bb680a9d4d 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause_test.go @@ -108,32 +108,36 @@ func TestEnsurePausedFinalized_RecordsContentScope(t *testing.T) { ctx := context.Background() actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} + workerName := testWorkerUID("worker-pod-1") created := storetest.MustCreateActor(t, ctx, st, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: actorRef.Atespace, Name: actorRef.Name}, Status: &ateapipb.ActorStatus{ State: ateapipb.ActorState_ACTOR_STATE_PAUSING, WorkerAssignment: &ateapipb.WorkerAssignment{ + Worker: &ateapipb.ObjectRef{Name: workerName}, WorkerNamespace: "default", WorkerPool: "pool1", WorkerPod: "worker-pod-1", + WorkerPodUid: workerName, }, InProgressLocalSnapshotName: "snap-prefix", }, }) if _, err := st.CreateWorker(ctx, &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: workerName}, WorkerNamespace: "default", WorkerPool: "pool1", WorkerPod: "worker-pod-1", + WorkerPodUid: workerName, NodeName: "node1", - Status: &ateapipb.WorkerStatus{ - Assignment: &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: actorRef.Atespace, Name: actorRef.Name}, - ActorUid: created.GetMetadata().GetUid(), - }, - }, + Status: &ateapipb.WorkerStatus{}, }); err != nil { t.Fatalf("CreateWorker: %v", err) } + seedAssignment(t, st, workerName, &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: actorRef.Atespace, Name: actorRef.Name}, + ActorUid: created.GetMetadata().GetUid(), + }) w := &ActorWorkflow{store: st} tmpl := &ateapipb.ActorTemplate{ diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index 0e8c62258e..64111a1f73 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -29,7 +29,6 @@ import ( "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "google.golang.org/protobuf/proto" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" @@ -324,11 +323,13 @@ func (w *ActorWorkflow) ensureWorkerAssigned(ctx context.Context, actorRef resou return nil, nil, status.Errorf(codes.FailedPrecondition, "AssignWorker prerequisite not met for Actor: %s (got: %v, want %s or %s)", actorRef, actor.GetStatus().GetState(), ateapipb.ActorState_ACTOR_STATE_SUSPENDED, ateapipb.ActorState_ACTOR_STATE_PAUSED) } + // Bound contention retries to about three seconds. backoff := wait.Backoff{ - Steps: 5, - Duration: 10 * time.Millisecond, + Steps: 12, + Duration: 15 * time.Millisecond, Factor: 2.0, Jitter: 1.0, + Cap: 250 * time.Millisecond, } var assignedActor *ateapipb.Actor var assignedWorker *ateapipb.Worker @@ -341,7 +342,7 @@ func (w *ActorWorkflow) ensureWorkerAssigned(ctx context.Context, actorRef resou assignedActor, assignedWorker = attemptActor, attemptWorker return true, nil } - if errors.Is(attemptErr, store.ErrVersionConflict) { + if errors.Is(attemptErr, store.ErrVersionConflict) || errors.Is(attemptErr, errWorkerFilledUp) { if attemptActor != nil { actor = attemptActor // retry with the refreshed actor } @@ -394,11 +395,14 @@ func (w *ActorWorkflow) validateAssignedWorker(ctx context.Context, actorRef res } return nil, status.Errorf(codes.Aborted, "actor %s crashed", actorRef.String()) } - // Verify the worker is still assigned to the same Actor. - if worker.GetStatus().GetAssignment().GetActorUid() != actor.GetMetadata().GetUid() { - slog.ErrorContext(ctx, "crashing actor because its assigned worker no longer belongs to it", - slog.String("worker", worker.GetWorkerPod()), - slog.Any("assignment", worker.GetStatus().GetAssignment())) + // Verify the worker is still hosting this Actor. + hosted, err := workerHostsActor(ctx, w.store, worker.GetMetadata().GetName(), actor.GetMetadata().GetUid()) + if err != nil { + return nil, err + } + if !hosted { + slog.ErrorContext(ctx, "crashing actor because its assigned worker no longer hosts it", + slog.String("worker", worker.GetWorkerPod())) if cerr := crashActor(ctx, w.store, actorRef, ateattr.OperationResume, ateattr.ReasonWorkerReassigned); cerr != nil { return nil, fmt.Errorf("while crashing actor: %w", cerr) } @@ -414,10 +418,7 @@ func (w *ActorWorkflow) validateAssignedWorker(ctx context.Context, actorRef res // worker_selector was updated after the failed attempt), release it back // to the free pool instead of leaving it claimed forever — nothing else // reclaims a healthy worker whose actor moved on to a different pool. - if _, err := w.store.UpdateWorker(ctx, worker.GetMetadata().GetName(), store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.Assignment = nil - return nil - }); err != nil { + if _, err := w.store.ReleaseActorFromWorker(ctx, worker.GetMetadata().GetName(), actor.GetMetadata().GetUid()); err != nil { return nil, fmt.Errorf("while releasing stale worker assignment: %w", err) } if cerr := crashActor(ctx, w.store, actorRef, ateattr.OperationResume, ateattr.ReasonCorruptedAssignment); cerr != nil { @@ -428,6 +429,43 @@ func (w *ActorWorkflow) validateAssignedWorker(ctx context.Context, actorRef res return worker, nil } +// admittedResources is what an assignment books against its worker, or nil +// when the actor declared no limits and so reserves nothing. +func admittedResources(constraints scheduling.Constraints) *ateapipb.Resources { + return constraints.Limits +} + +// workerHoldingStaleClaim recovers a claim written before the Actor update. +// It releases the claim if the Worker is no longer eligible. +func (w *ActorWorkflow) workerHoldingStaleClaim(ctx context.Context, actor *ateapipb.Actor, constraints scheduling.Constraints) (*ateapipb.Worker, error) { + actorUID := actor.GetMetadata().GetUid() + workerName, err := w.store.FindWorkerHostingActor(ctx, actorUID) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return nil, nil + } + return nil, fmt.Errorf("while looking for a worker already hosting actor %q: %w", actorUID, err) + } + worker, err := w.store.GetWorker(ctx, workerName) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return nil, nil // the worker went away with its claim + } + return nil, fmt.Errorf("while reading worker %q holding a stale claim: %w", workerName, err) + } + + // The Actor's allocation makes HasRoom unsuitable for an existing claim. + if w.scheduler.Applies(worker, constraints) { + return worker, nil + } + + _, err = w.store.ReleaseActorFromWorker(ctx, workerName, actorUID) + if err != nil { + return nil, fmt.Errorf("while releasing stale claim on worker %q: %w", workerName, err) + } + return nil, nil +} + // schedulerRecordable excludes retried version conflicts: the assignment loop // re-runs attempts transparently on store.ErrVersionConflict, so counting // those attempts would inflate the error rate and double-count the eventual @@ -456,52 +494,14 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour } }() - workers, err := w.workerCache.Workers() - if err != nil { - return nil, nil, fmt.Errorf("while listing workers: %w", err) - } - constraints, err := schedulingConstraints(actor, actorTemplate) if err != nil { return nil, nil, err } - var assignedWorker *ateapipb.Worker - - // Check if we already have a worker assigned from a previous failed attempt. - // This can happen if ateapi crashed after updating worker with actor assignment, - // but has not yet updated the actor. - for _, worker := range workers { - if worker.GetStatus().GetAssignment() == nil { - continue - } - if worker.GetStatus().GetAssignment().GetActorUid() != actor.GetMetadata().GetUid() { - continue - } - if w.scheduler.Applies(worker, constraints) { - assignedWorker = worker - break - } - // Workers() returns pointers directly from the cache, so clone before - // handing the worker to the goroutine: the mutation runs against the - // store's own copy, but the precondition and the log below read this one. - releaseWorker := proto.Clone(worker).(*ateapipb.Worker) - // The claimed worker is no longer eligible (e.g. the actor's - // worker_selector changed after the failed attempt); release it back - // to the free pool — nothing else reclaims a healthy worker whose - // actor moved on to a different pool. Best effort in the background. - go func(release *ateapipb.Worker) { - bgCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if _, err := w.store.UpdateWorker(bgCtx, release.GetMetadata().GetName(), store.PreconditionFrom(release), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.Assignment = nil - return nil - }); err != nil { - slog.ErrorContext(bgCtx, "Failed to release stale worker assignment", - slog.String("worker", release.GetWorkerNamespace()+"/"+release.GetWorkerPod()), - slog.Any("err", err)) - } - }(releaseWorker) + assignedWorker, err := w.workerHoldingStaleClaim(ctx, actor, constraints) + if err != nil { + return nil, nil, err } if assignedWorker == nil { pickedWorker, err := w.scheduler.Schedule(ctx, constraints) @@ -523,24 +523,28 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour Name: actor.GetMetadata().GetName(), }, ActorUid: actor.GetMetadata().GetUid(), + // Record what this claim reserves so release returns the same amount. + Resources: admittedResources(constraints), } assignment.ActorTemplateRef = actorTemplateObjectRef(actor) - // Workers() returns pointers directly from the cache, so the claim is written - // by mutating the store's own copy; the cached one is only read, for the - // version this claim is conditioned on. - stored, err := w.store.UpdateWorker(ctx, assignedWorker.GetMetadata().GetName(), store.PreconditionFrom(assignedWorker), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.Assignment = assignment + // The candidate came from a watch-fed cache, so it may already be full or no + // longer eligible. The store re-asks under the Worker's row lock, where the + // answer holds until the bind commits, so nothing here needs a fresh read + // and two claims for the last place cannot both be admitted. + admit := func(fresh *ateapipb.Worker) error { + if !w.scheduler.Applies(fresh, constraints) || !w.scheduler.HasRoom(fresh, constraints) { + return errWorkerFilledUp + } return nil - }) - if err != nil { + } + if err := w.store.BindActorToWorker(ctx, assignedWorker.GetMetadata().GetName(), assignment, admit); err != nil { if errors.Is(err, store.ErrNotFound) { w.workerCache.Forget(assignedWorker.GetMetadata().GetName()) return nil, nil, fmt.Errorf("selected worker disappeared before claim: %w", store.ErrVersionConflict) } return nil, nil, err } - assignedWorker = stored newAssignment := workerAssignmentFrom(assignedWorker) storedActor, err := w.store.UpdateActor(ctx, actorRef, store.PreconditionFrom(actor), func(toUpdate *ateapipb.Actor) error { @@ -572,6 +576,11 @@ func (w *ActorWorkflow) assignWorkerAttempt(ctx context.Context, actorRef resour return storedActor, assignedWorker, nil } +// errWorkerFilledUp reports that the Worker the scheduler picked would not take +// the Actor once the store asked under its row lock. Retryable: the next +// attempt re-runs scheduling. +var errWorkerFilledUp = errors.New("picked worker no longer has room") + func workerAssignmentFrom(w *ateapipb.Worker) *ateapipb.WorkerAssignment { return &ateapipb.WorkerAssignment{ Worker: &ateapipb.ObjectRef{Name: w.GetMetadata().GetName()}, @@ -585,8 +594,8 @@ func workerAssignmentFrom(w *ateapipb.Worker) *ateapipb.WorkerAssignment { // actorResourceLimits returns the actor's declared CPU (millicores) and memory // (bytes) limits from its ActorTemplate, or 0 for a dimension the template did -// not set. These size the sandbox (supplied over the actor RPCs) and gate -// scheduling (a worker must have >= capacity). +// not set. These size the sandbox, which takes the two scalars the runtimes +// understand rather than the named set placement accounts in. func actorResourceLimits(tmpl *ateapipb.ActorTemplate) (cpuMilli, memBytes int64, err error) { for _, limit := range tmpl.GetResources().GetLimits() { q, perr := resource.ParseQuantity(limit.GetQuantity()) @@ -604,16 +613,17 @@ func actorResourceLimits(tmpl *ateapipb.ActorTemplate) (cpuMilli, memBytes int64 } func schedulingConstraints(actor *ateapipb.Actor, tmpl *ateapipb.ActorTemplate) (scheduling.Constraints, error) { - cpuMilli, memBytes, err := actorResourceLimits(tmpl) + // Canonicalized here, the one place an assignment's booked resources are + // decided, so what is recorded is sorted however the template was authored. + limits, err := resources.ParseQuantities(tmpl.GetResources()) if err != nil { - return scheduling.Constraints{}, err + return scheduling.Constraints{}, fmt.Errorf("invalid template resource limits: %w", err) } c := scheduling.Constraints{ SandboxClass: sandboxClassString(tmpl.GetSandboxConfig().GetSandboxClass()), ActorSelector: labels.SelectorFromSet(labels.Set(actor.GetWorkerSelector().GetMatchLabels())), RequiredNodes: actor.GetStatus().GetLocalSnapshotInfo().GetNodeVmsWithLocalSnapshots(), - CPUMilli: cpuMilli, - MemoryBytes: memBytes, + Limits: limits.Proto(), } if sel := tmpl.GetWorkerSelector(); sel != nil { c.TemplateSelector = labels.SelectorFromSet(labels.Set(sel.GetMatchLabels())) diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index bd54e4bd8f..65c18924ce 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -128,20 +128,22 @@ func TestFinalizeRunning_RecordsSprintTemplate(t *testing.T) { } } -type updateWorkerErrorStore struct { +// bindErrorStore fails every claim, standing in for a worker that moved or +// vanished between the pick and the write. +type bindErrorStore struct { store.Interface err error } -func (s *updateWorkerErrorStore) UpdateWorker(context.Context, string, store.Precondition, func(*ateapipb.Worker) error) (*ateapipb.Worker, error) { - return nil, s.err +func (s *bindErrorStore) BindActorToWorker(context.Context, string, *ateapipb.ActorAssignment, func(*ateapipb.Worker) error) error { + return s.err } func TestAssignWorkerAttempt_MissingSelectedWorkerIsRetried(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) actor, wc := seedAssignFixture(t, ctx, persistence) - st := &updateWorkerErrorStore{Interface: persistence, err: store.ErrNotFound} + st := &bindErrorStore{Interface: persistence, err: store.ErrNotFound} w := &ActorWorkflow{store: st, workerCache: wc, scheduler: scheduling.New(wc)} tmpl := &ateapipb.ActorTemplate{SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR}} @@ -162,7 +164,7 @@ func TestEnsureWorkerAssigned_ConflictExhaustionIsRetryable(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) actor, wc := seedAssignFixture(t, ctx, persistence) - st := &updateWorkerErrorStore{Interface: persistence, err: store.ErrVersionConflict} + st := &bindErrorStore{Interface: persistence, err: store.ErrVersionConflict} w := &ActorWorkflow{store: st, workerCache: wc, scheduler: scheduling.New(wc)} tmpl := &ateapipb.ActorTemplate{SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR}} @@ -186,7 +188,7 @@ func TestAssignWorkerAttempt_StampsSubstrateTemplateRef(t *testing.T) { WorkerPod: "pod-free", WorkerPodUid: testWorkerUID("pod-free"), SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE}, + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}}}, } if _, err := persistence.CreateWorker(ctx, worker); err != nil { t.Fatalf("CreateWorker: %v", err) @@ -215,11 +217,10 @@ func TestAssignWorkerAttempt_StampsSubstrateTemplateRef(t *testing.T) { t.Fatalf("assignWorkerAttempt: %v", err) } - stored, err := persistence.GetWorker(ctx, assigned.GetMetadata().GetName()) + assignment, err := persistence.GetWorkerAssignment(ctx, assigned.GetMetadata().GetName(), actor.GetMetadata().GetUid()) if err != nil { - t.Fatalf("GetWorker: %v", err) + t.Fatalf("GetWorkerAssignment: %v", err) } - assignment := stored.GetStatus().GetAssignment() if assignment.GetActorTemplateRef().GetAtespace() != "team-a" || assignment.GetActorTemplateRef().GetName() != "sub-tmpl" { t.Errorf("assignment ActorTemplateRef = %v, want team-a/sub-tmpl", assignment.GetActorTemplateRef()) } @@ -238,17 +239,15 @@ func TestAssignWorkerAttempt_SkipsWorkerAssignedInOtherAtespace(t *testing.T) { WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Assignment: &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: "team-b", Name: "shared"}, - ActorUid: "team-b-actor-uid", - }, - }, + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}}}, } if _, err := persistence.CreateWorker(ctx, worker); err != nil { t.Fatalf("CreateWorker: %v", err) } + seedAssignment(t, persistence, testWorkerUID("pod-1"), &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: "team-b", Name: "shared"}, + ActorUid: "team-b-actor-uid", + }) cacheCtx, cancel := context.WithCancel(ctx) defer cancel() @@ -269,23 +268,20 @@ func TestAssignWorkerAttempt_SkipsWorkerAssignedInOtherAtespace(t *testing.T) { t.Fatalf("assignWorkerAttempt() error = %v, want ResourceExhausted (no free workers)", err) } - stored, err := persistence.GetWorker(ctx, testWorkerUID("pod-1")) - if err != nil { - t.Fatalf("GetWorker: %v", err) - } - if got := stored.GetStatus().GetAssignment().GetActorUid(); got != "team-b-actor-uid" { - t.Errorf("worker assignment uid = %q, want %q (assignment: %v)", got, "team-b-actor-uid", stored.GetStatus().GetAssignment()) + stored := firstAssignment(t, persistence, testWorkerUID("pod-1")) + if got := stored.GetActorUid(); got != "team-b-actor-uid" { + t.Errorf("worker assignment uid = %q, want %q (assignment: %v)", got, "team-b-actor-uid", stored) } - if got := stored.GetStatus().GetAssignment().GetActor().GetAtespace(); got != "team-b" { - t.Errorf("worker assignment atespace = %q, want %q (assignment: %v)", got, "team-b", stored.GetStatus().GetAssignment()) + if got := stored.GetActor().GetAtespace(); got != "team-b" { + t.Errorf("worker assignment atespace = %q, want %q (assignment: %v)", got, "team-b", stored) } } -// TestAssignWorkerAttempt_ReleasesIneligibleStaleWorkerInBackground verifies -// that a worker claimed by a previous failed attempt whose pool is no longer -// eligible is released back to the free pool asynchronously, without failing -// the resume, while a fresh eligible worker is assigned. -func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorkerInBackground(t *testing.T) { +// TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker verifies that a worker +// claimed by a previous failed attempt whose pool is no longer eligible is +// released back to the free pool, without failing the resume, while a fresh +// eligible worker is assigned. +func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorker(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) @@ -303,13 +299,7 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorkerInBackground(t *testin WorkerPod: "stale-pod", WorkerPodUid: testWorkerUID("stale-pod"), SandboxClass: "microvm", - Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Assignment: &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "id1"}, - ActorUid: actor.GetMetadata().GetUid(), - }, - }, + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}}}, } free := &ateapipb.Worker{ Metadata: &ateapipb.ResourceMetadata{Name: testWorkerUID("free-pod")}, @@ -318,15 +308,17 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorkerInBackground(t *testin WorkerPod: "free-pod", WorkerPodUid: testWorkerUID("free-pod"), SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - }, + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}}}, } for _, w := range []*ateapipb.Worker{stale, free} { if _, err := persistence.CreateWorker(ctx, w); err != nil { t.Fatalf("CreateWorker(%s): %v", w.GetWorkerPod(), err) } } + seedAssignment(t, persistence, testWorkerUID("stale-pod"), &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "id1"}, + ActorUid: actor.GetMetadata().GetUid(), + }) cacheCtx, cancel := context.WithCancel(ctx) defer cancel() @@ -348,21 +340,10 @@ func TestAssignWorkerAttempt_ReleasesIneligibleStaleWorkerInBackground(t *testin t.Errorf("assigned worker = %q, want %q", got, "free-pod") } - // The stale worker is released in the background; poll until its - // assignment is cleared. - deadline := time.Now().Add(5 * time.Second) - for { - stored, err := persistence.GetWorker(ctx, testWorkerUID("stale-pod")) - if err != nil { - t.Fatalf("GetWorker: %v", err) - } - if stored.GetStatus().GetAssignment() == nil { - break - } - if time.Now().After(deadline) { - t.Fatalf("stale worker still assigned after %v: %v", 5*time.Second, stored.GetStatus().GetAssignment()) - } - time.Sleep(10 * time.Millisecond) + // The stale worker must already be released: the actor could not have been + // placed on another worker otherwise. + if stored := firstAssignment(t, persistence, testWorkerUID("stale-pod")); stored != nil { + t.Errorf("stale worker still assigned: %v", stored) } } @@ -382,9 +363,7 @@ func TestAssignWorkerAttempt_RetryAfterConflictPicksFreshWorker(t *testing.T) { WorkerPod: "contested-pod", WorkerPodUid: testWorkerUID("contested-pod"), SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - }, + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}}}, } fallback := &ateapipb.Worker{ Metadata: &ateapipb.ResourceMetadata{Name: testWorkerUID("fallback-pod")}, @@ -393,9 +372,7 @@ func TestAssignWorkerAttempt_RetryAfterConflictPicksFreshWorker(t *testing.T) { WorkerPod: "fallback-pod", WorkerPodUid: testWorkerUID("fallback-pod"), SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - }, + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}}}, } for _, w := range []*ateapipb.Worker{contested, fallback} { if _, err := persistence.CreateWorker(ctx, w); err != nil { @@ -403,23 +380,12 @@ func TestAssignWorkerAttempt_RetryAfterConflictPicksFreshWorker(t *testing.T) { } } - // Snapshot the contested worker at the version the failed attempt saw. - beforeClaim, err := persistence.GetWorker(ctx, testWorkerUID("contested-pod")) - if err != nil { - t.Fatalf("GetWorker: %v", err) - } - // A concurrent resume of another actor wins the contested worker, bumping // its stored version past the failed attempt's snapshot. - if _, err := persistence.UpdateWorker(ctx, beforeClaim.GetMetadata().GetName(), store.PreconditionFrom(beforeClaim), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.Assignment = &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "other"}, - ActorUid: "other-actor-uid", - } - return nil - }); err != nil { - t.Fatalf("UpdateWorker (concurrent claim): %v", err) - } + seedAssignment(t, persistence, testWorkerUID("contested-pod"), &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "other"}, + ActorUid: "other-actor-uid", + }) actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "id1"}, @@ -445,19 +411,13 @@ func TestAssignWorkerAttempt_RetryAfterConflictPicksFreshWorker(t *testing.T) { t.Errorf("assigned worker = %q, want %q", got, "fallback-pod") } - storedContested, err := persistence.GetWorker(ctx, testWorkerUID("contested-pod")) - if err != nil { - t.Fatalf("GetWorker(contested-pod): %v", err) + storedContested := firstAssignment(t, persistence, testWorkerUID("contested-pod")) + if got := storedContested.GetActorUid(); got != "other-actor-uid" { + t.Errorf("contested worker assignment = %v, want to remain with actor %q", storedContested, "other-actor-uid") } - if got := storedContested.GetStatus().GetAssignment().GetActorUid(); got != "other-actor-uid" { - t.Errorf("contested worker assignment = %v, want to remain with actor %q", storedContested.GetStatus().GetAssignment(), "other-actor-uid") - } - storedFallback, err := persistence.GetWorker(ctx, testWorkerUID("fallback-pod")) - if err != nil { - t.Fatalf("GetWorker(fallback-pod): %v", err) - } - if got := storedFallback.GetStatus().GetAssignment().GetActorUid(); got != actor.GetMetadata().GetUid() { - t.Errorf("fallback worker assignment = %v, want actor uid %q", storedFallback.GetStatus().GetAssignment(), actor.GetMetadata().GetUid()) + storedFallback := firstAssignment(t, persistence, testWorkerUID("fallback-pod")) + if got := storedFallback.GetActorUid(); got != actor.GetMetadata().GetUid() { + t.Errorf("fallback worker assignment = %v, want actor uid %q", storedFallback, actor.GetMetadata().GetUid()) } storedActor, err := persistence.GetActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}) @@ -502,9 +462,7 @@ func seedAssignFixture(t *testing.T, ctx context.Context, persistence store.Inte WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - }, + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}}}, }); err != nil { t.Fatalf("CreateWorker: %v", err) } @@ -877,13 +835,11 @@ func TestValidateAssignedWorker_WorkerOwnership(t *testing.T) { WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), SandboxClass: tt.sandboxClass, - Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - Assignment: tt.assignment, - }, + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}}}, }); err != nil { t.Fatalf("CreateWorker: %v", err) } + seedAssignment(t, persistence, testWorkerUID("pod-1"), tt.assignment) // Fetch the stored version so the no-write assertion below can // detect any optimistic update. seeded, err := persistence.GetWorker(ctx, testWorkerUID("pod-1")) @@ -925,8 +881,8 @@ func TestValidateAssignedWorker_WorkerOwnership(t *testing.T) { if err != nil { t.Fatalf("GetWorker: %v", err) } - if !proto.Equal(stored.GetStatus().GetAssignment(), tt.wantAssignment) { - t.Errorf("stored worker assignment = %v, want %v", stored.GetStatus().GetAssignment(), tt.wantAssignment) + if got := firstAssignment(t, persistence, testWorkerUID("pod-1")); !proto.Equal(got, tt.wantAssignment) { + t.Errorf("stored worker assignment = %v, want %v", got, tt.wantAssignment) } if !tt.wantWorkerWrite && stored.GetMetadata().GetVersion() != seeded.GetMetadata().GetVersion() { t.Errorf("worker version moved %d -> %d, want no write", seeded.GetMetadata().GetVersion(), stored.GetMetadata().GetVersion()) diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index 07f61d77be..424268a6ae 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -374,7 +374,7 @@ func (w *ActorWorkflow) ensureSuspendedFinalized(ctx context.Context, actorRef r // 1. Free the worker (if it hasn't been freed yet) if latestActor.GetStatus().GetWorkerAssignment() != nil { t = time.Now() - _, err := releaseWorker(ctx, w.store, latestActor) + _, _, err := releaseWorker(ctx, w.store, latestActor) dReleaseWorker = time.Since(t) if err != nil { return nil, err diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go index e7dfdf659a..2cfaecb6a8 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend_test.go @@ -436,16 +436,15 @@ func TestEnsureSuspendedFinalized_ReleasesOnlyOwnWorker(t *testing.T) { WorkerPool: "pool", WorkerPod: "pod-1", WorkerPodUid: testWorkerUID("pod-1"), - Status: &ateapipb.WorkerStatus{ - Assignment: &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: tt.assignmentAtespace, Name: "shared"}, - ActorUid: uid, - }, - }, + Status: &ateapipb.WorkerStatus{}, } if _, err := persistence.CreateWorker(ctx, worker); err != nil { t.Fatalf("CreateWorker: %v", err) } + seedAssignment(t, persistence, testWorkerUID("pod-1"), &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: tt.assignmentAtespace, Name: "shared"}, + ActorUid: uid, + }) w := &ActorWorkflow{store: persistence} tmpl := &ateapipb.ActorTemplate{SnapshotsConfig: &ateapipb.SnapshotsConfig{StorageLocation: "gs://bucket/root"}} @@ -453,12 +452,9 @@ func TestEnsureSuspendedFinalized_ReleasesOnlyOwnWorker(t *testing.T) { t.Fatalf("ensureSuspendedFinalized: %v", err) } - stored, err := persistence.GetWorker(ctx, testWorkerUID("pod-1")) - if err != nil { - t.Fatalf("GetWorker: %v", err) - } - if released := stored.GetStatus().GetAssignment() == nil; released != tt.wantReleased { - t.Errorf("worker released = %t, want %t (assignment: %v)", released, tt.wantReleased, stored.GetStatus().GetAssignment()) + stored := firstAssignment(t, persistence, testWorkerUID("pod-1")) + if released := stored == nil; released != tt.wantReleased { + t.Errorf("worker released = %t, want %t (assignment: %v)", released, tt.wantReleased, stored) } }) } diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go index ec6071da73..d37cc0e4c6 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete.go @@ -42,10 +42,35 @@ func (w *WorkerWorkflow) DeleteWorker(ctx context.Context, name string, pre stor return nil, err } + // Checked against the Worker the caller observed, before the drain below + // moves the version. + if err := pre.Check(worker.GetMetadata()); err != nil { + switch { + case errors.Is(err, store.ErrUIDConflict): + return nil, status.Errorf(codes.Aborted, "Worker %s does not have uid %s", name, pre.UID) + case errors.Is(err, store.ErrVersionConflict): + return nil, status.Error(codes.Aborted, "concurrent update conflict, please retry") + } + return nil, err + } + + // Scheduling only places on ACTIVE Workers, so draining first stops a + // concurrent resume from binding to a page the sweep has already passed. + // The delete would cascade that assignment away and leave the Actor + // pointing at a Worker that is gone. + worker, err = w.ensureDraining(ctx, worker) + if err != nil { + return nil, err + } + + // The drain moved the version, so only the uid guard still means anything: + // a Worker replaced by a new incarnation mid-delete is still refused. + pre.Version = 0 + // Order matters: the delete is what erases the Actor's pointer at the // Worker, so a failed release has to leave the record in place for the // caller to rediscover and retry. - if err := w.ensureBoundActorReleased(ctx, worker); err != nil { + if err := w.ensureBoundActorsReleased(ctx, worker); err != nil { return nil, err } @@ -68,29 +93,82 @@ func (w *WorkerWorkflow) loadWorkerForDelete(ctx context.Context, name string) ( return worker, nil } -// ensureBoundActorReleased resets the Actor bound to the Worker. An Actor that -// already reached ACTOR_STATE_SUSPENDED saved its state cleanly during graceful +// ensureDraining moves the Worker out of the state scheduling will place on, +// and is a no-op for one already draining. +func (w *WorkerWorkflow) ensureDraining(ctx context.Context, worker *ateapipb.Worker) (_ *ateapipb.Worker, err error) { + ctx, done := stepSpan(ctx, "DrainWorkerForDelete") + defer func() { err = done(err) }() + + if worker.GetStatus().GetState() == ateapipb.WorkerState_WORKER_STATE_DRAINING { + return worker, nil + } + drained, err := w.store.UpdateWorker(ctx, worker.GetMetadata().GetName(), store.PreconditionFrom(worker), + func(toUpdate *ateapipb.Worker) error { + if toUpdate.Status == nil { + toUpdate.Status = &ateapipb.WorkerStatus{} + } + toUpdate.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING + return nil + }) + if err != nil { + return nil, fmt.Errorf("while draining worker for delete: %w", err) + } + return drained, nil +} + +// ensureBoundActorsReleased resets every Actor bound to the Worker. +// +// A single failure stops the sweep, leaving the Worker record in place with the +// Actors that have not been released still bound to it, so a retry picks up +// where this left off. +func (w *WorkerWorkflow) ensureBoundActorsReleased(ctx context.Context, worker *ateapipb.Worker) (err error) { + ctx, done := stepSpan(ctx, "ReleaseBoundActors") + defer func() { err = done(err) }() + + // Every page: a Worker can hold thousands of Actors and a page holds at + // most a thousand, so stopping at the first would leave the rest bound. + var released int + for token := ""; ; { + page, err := w.store.ListWorkerAssignments(ctx, worker.GetMetadata().GetName(), store.ListOptions{PageToken: token}) + if err != nil { + return fmt.Errorf("while listing the assignments of worker %s: %w", worker.GetMetadata().GetName(), err) + } + for _, assignment := range page.Items { + if err := w.releaseBoundActor(ctx, worker, assignment); err != nil { + return err + } + released++ + } + if !page.HasNextPage() { + break + } + token = page.NextPageToken + } + if released == 0 { + markSkipped(ctx, "worker has no actors assigned") + } + return nil +} + +// releaseBoundActor resets one Actor bound to the Worker. An Actor that already +// reached ACTOR_STATE_SUSPENDED saved its state cleanly during graceful // termination, so it is left untouched and remains resumable. An Actor that was // still running when the pod disappeared is moved to ACTOR_STATE_CRASHED and its // pod pointers are cleared. // -// Nothing to release is the common case and reports success: an unassigned -// Worker, a superseded assignment, and an Actor that has since moved elsewhere -// all leave no Actor pointing at this Worker, which is the state this is driving -// towards. +// Nothing to release is the common case and reports success: a superseded +// assignment and an Actor that has since moved elsewhere both leave no Actor +// pointing at this Worker, which is the state this is driving towards. // // A concurrent SuspendActor or ResumeActor wins the optimistic version check; // this attempt fails as ABORTED so the caller retries against the newer state. -func (w *WorkerWorkflow) ensureBoundActorReleased(ctx context.Context, worker *ateapipb.Worker) (err error) { - ctx, done := stepSpan(ctx, "ReleaseBoundActor") - defer func() { err = done(err) }() - - if worker.GetStatus().GetAssignment().GetActor() == nil { - markSkipped(ctx, "worker has no actor assigned") +func (w *WorkerWorkflow) releaseBoundActor(ctx context.Context, worker *ateapipb.Worker, assignment *ateapipb.ActorAssignment) error { + if assignment.GetActor() == nil { + markSkipped(ctx, "assignment names no actor") return nil } name := worker.GetMetadata().GetName() - actorRef := resources.ActorRefFromObjectRef(worker.GetStatus().GetAssignment().GetActor()) + actorRef := resources.ActorRefFromObjectRef(assignment.GetActor()) actor, err := w.store.GetActor(ctx, actorRef) if errors.Is(err, store.ErrNotFound) { markSkipped(ctx, "assigned actor no longer exists") @@ -99,7 +177,7 @@ func (w *WorkerWorkflow) ensureBoundActorReleased(ctx context.Context, worker *a if err != nil { return fmt.Errorf("while getting actor to release from worker %s: %w", name, err) } - if actor.GetMetadata().GetUid() != worker.GetStatus().GetAssignment().GetActorUid() { + if actor.GetMetadata().GetUid() != assignment.GetActorUid() { markSkipped(ctx, "assignment names a superseded actor incarnation") return nil } diff --git a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go index 4b318c0316..2914426143 100644 --- a/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_worker_delete_test.go @@ -76,6 +76,32 @@ func seedAPIActor(t *testing.T, ctx context.Context, persistence store.Interface // it with it. The release happens in this workflow rather than in the caller // that noticed the pod had vanished, because an assignment write stays // in-process: there is no bind/release RPC for that caller to reach for. +// A Worker left ACTIVE through the sweep can take a bind onto a page already +// passed, and the delete then cascades that assignment away while the Actor +// still points at it. Failing the release keeps the record around, so the +// state it held during the sweep can be read. +func TestDeleteWorkerWorkflow_DrainsBeforeSweeping(t *testing.T) { + ctx := context.Background() + _, persistence := newWorkerDeleteWorkflow(t) + seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) + actor := seedAPIActor(t, ctx, persistence, ateapipb.ActorState_ACTOR_STATE_RUNNING) + assignAPIWorker(t, ctx, persistence, apiWorkerName, actor.GetMetadata().GetUid()) + + wf := NewWorkerWorkflow(failingUpdateActorStore{Interface: persistence, err: errors.New("release failed")}) + if _, err := wf.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}); err == nil { + t.Fatal("DeleteWorker() = nil error, want the release failure reported") + } + + got, err := persistence.GetWorker(ctx, apiWorkerName) + if err != nil { + t.Fatalf("GetWorker() failed: %v", err) + } + if got.GetStatus().GetState() != ateapipb.WorkerState_WORKER_STATE_DRAINING { + t.Errorf("worker state during the sweep = %v, want DRAINING so nothing new binds", + got.GetStatus().GetState()) + } +} + func TestDeleteWorkerWorkflow_ReleasesBoundActor(t *testing.T) { ctx := context.Background() wf, persistence := newWorkerDeleteWorkflow(t) @@ -228,12 +254,13 @@ func TestDeleteWorkerWorkflow_AssignedToAbsentActorDeletesAnyway(t *testing.T) { seedAPIWorker(t, ctx, persistence, validWorker(apiWorkerName)) assignAPIWorker(t, ctx, persistence, apiWorkerName, "actor-uid-1") - got, err := wf.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}) - if err != nil { + if _, err := wf.DeleteWorker(ctx, apiWorkerName, store.DeletePreconditions{}); err != nil { t.Fatalf("DeleteWorker() failed: %v", err) } - if got.GetStatus().GetAssignment().GetActorUid() != "actor-uid-1" { - t.Errorf("deleted worker assignment = %v, want the one it was holding", got.GetStatus().GetAssignment()) + // An assignment is its own record now, so what proves the delete went + // through is that nothing is left pointing at the Worker. + if got := firstAssignment(t, persistence, apiWorkerName); got != nil { + t.Errorf("assignment after delete = %v, want none", got) } } diff --git a/cmd/ateapi/internal/controlapi/zz_generated.validation.go b/cmd/ateapi/internal/controlapi/zz_generated.validation.go index 4fffe646e4..30729db4ec 100644 --- a/cmd/ateapi/internal/controlapi/zz_generated.validation.go +++ b/cmd/ateapi/internal/controlapi/zz_generated.validation.go @@ -245,139 +245,6 @@ func Validate_Actor( return errs } -// Validate_ActorAssignment validates an instance of ActorAssignment according -// to declarative validation rules in the API schema. -func Validate_ActorAssignment( - ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateapipb.ActorAssignment) (errs field.ErrorList) { - - { // field ateapipb.ActorAssignment.Actor - fn := func( - fldPath *field.Path, - obj, oldObj *ateapipb.ObjectRef, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - func() { // cohort = "atespace" - earlyReturn := false - if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", - func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.RequiredValue).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", - func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - }() - // call the type's validation function - errs = append(errs, Validate_ObjectRef(ctx, op, fldPath, obj, oldObj)...) - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorAssignment) *ateapipb.ObjectRef { - return oldObj.Actor - }) - errs = append(errs, fn(fldPath.Child("actor"), obj.Actor, oldVal, oldObj != nil)...) - } - - { // field ateapipb.ActorAssignment.ActorUid - fn := func( - fldPath *field.Path, - obj, oldObj *string, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - if e := validate.UUID(ctx, op, fldPath, obj, oldObj); len(e) != 0 { - errs = append(errs, e...) - } - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorAssignment) *string { - return &oldObj.ActorUid - }) - errs = append(errs, fn(fldPath.Child("actor_uid"), &obj.ActorUid, oldVal, oldObj != nil)...) - } - - { // field ateapipb.ActorAssignment.ActorTemplateRef - fn := func( - fldPath *field.Path, - obj, oldObj *ateapipb.ObjectRef, - oldValueCorrelated bool) (errs field.ErrorList) { - // don't revalidate unchanged data - if oldValueCorrelated && op.Type == operation.Update { - if ateDeepEqual(obj, oldObj) { - return nil - } - } - // call field-attached validations - earlyReturn := false - if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - func() { // cohort = "atespace" - earlyReturn := false - if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", - func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.RequiredValue).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) - earlyReturn = true - } - if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", - func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkShortCircuit(); len(e) != 0 { - earlyReturn = true - } - if earlyReturn { - return // do not proceed - } - }() - // call the type's validation function - errs = append(errs, Validate_ObjectRef(ctx, op, fldPath, obj, oldObj)...) - return - } - oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.ActorAssignment) *ateapipb.ObjectRef { - return oldObj.ActorTemplateRef - }) - errs = append(errs, fn(fldPath.Child("actor_template_ref"), obj.ActorTemplateRef, oldVal, oldObj != nil)...) - } - - return errs -} - // Validate_ActorMetadataDataSource validates an instance of ActorMetadataDataSource according // to declarative validation rules in the API schema. func Validate_ActorMetadataDataSource( @@ -4199,6 +4066,127 @@ func Validate_ListAtespacesRequest( return errs } +// Validate_ListWorkerActorAssignmentsRequest validates an instance of ListWorkerActorAssignmentsRequest according +// to declarative validation rules in the API schema. +func Validate_ListWorkerActorAssignmentsRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.ListWorkerActorAssignmentsRequest) (errs field.ErrorList) { + + { // field ateapipb.ListWorkerActorAssignmentsRequest.Worker + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.ObjectRef, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + func() { // cohort = "atespace" + earlyReturn := false + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.ForbiddenValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + }() + // call the type's validation function + errs = append(errs, Validate_ObjectRef(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.ListWorkerActorAssignmentsRequest) *ateapipb.ObjectRef { + return oldObj.Worker + }) + errs = append(errs, fn(fldPath.Child("worker"), obj.Worker, oldVal, oldObj != nil)...) + } + + { // field ateapipb.ListWorkerActorAssignmentsRequest.PageSize + fn := func( + fldPath *field.Path, + obj, oldObj *int32, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.ListWorkerActorAssignmentsRequest) *int32 { + return &oldObj.PageSize + }) + errs = append(errs, fn(fldPath.Child("page_size"), &obj.PageSize, oldVal, oldObj != nil)...) + } + + { // field ateapipb.ListWorkerActorAssignmentsRequest.PageToken + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.MaxLength(ctx, op, fldPath, obj, oldObj, 256); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.ListWorkerActorAssignmentsRequest) *string { + return &oldObj.PageToken + }) + errs = append(errs, fn(fldPath.Child("page_token"), &obj.PageToken, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_ListWorkersRequest validates an instance of ListWorkersRequest according // to declarative validation rules in the API schema. func Validate_ListWorkersRequest( @@ -5396,6 +5384,96 @@ func Validate_Selector( return errs } +// Validate_SetWorkerCapacityRequest validates an instance of SetWorkerCapacityRequest according +// to declarative validation rules in the API schema. +func Validate_SetWorkerCapacityRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.SetWorkerCapacityRequest) (errs field.ErrorList) { + + { // field ateapipb.SetWorkerCapacityRequest.Worker + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.ObjectRef, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + func() { // cohort = "atespace" + earlyReturn := false + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.ForbiddenValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkBeta().MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + }() + // call the type's validation function + errs = append(errs, Validate_ObjectRef(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SetWorkerCapacityRequest) *ateapipb.ObjectRef { + return oldObj.Worker + }) + errs = append(errs, fn(fldPath.Child("worker"), obj.Worker, oldVal, oldObj != nil)...) + } + + { // field ateapipb.SetWorkerCapacityRequest.Capacity + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.WorkerResources, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_WorkerResources(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SetWorkerCapacityRequest) *ateapipb.WorkerResources { + return oldObj.Capacity + }) + errs = append(errs, fn(fldPath.Child("capacity"), obj.Capacity, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_SnapshotsConfig validates an instance of SnapshotsConfig according // to declarative validation rules in the API schema. func Validate_SnapshotsConfig( @@ -6614,10 +6692,10 @@ func Validate_Worker( errs = append(errs, fn(fldPath.Child("labels"), obj.Labels, oldVal, oldObj != nil)...) } - { // field ateapipb.Worker.Capacity + { // field ateapipb.Worker.Status fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.WorkerCapacity, + obj, oldObj *ateapipb.WorkerStatus, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -6627,10 +6705,45 @@ func Validate_Worker( } // call field-attached validations earlyReturn := false - if e := validate.Immutable(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { - errs = append(errs, e...) + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_WorkerStatus(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.Worker) *ateapipb.WorkerStatus { + return oldObj.Status + }) + errs = append(errs, fn(fldPath.Child("status"), obj.Status, oldVal, oldObj != nil)...) + } + + return errs +} + +// Validate_WorkerAllocation validates an instance of WorkerAllocation according +// to declarative validation rules in the API schema. +func Validate_WorkerAllocation( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.WorkerAllocation) (errs field.ErrorList) { + + { // field ateapipb.WorkerAllocation.Capacity + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.WorkerResources, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } @@ -6638,20 +6751,20 @@ func Validate_Worker( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_WorkerCapacity(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_WorkerResources(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.Worker) *ateapipb.WorkerCapacity { + func(oldObj *ateapipb.WorkerAllocation) *ateapipb.WorkerResources { return oldObj.Capacity }) errs = append(errs, fn(fldPath.Child("capacity"), obj.Capacity, oldVal, oldObj != nil)...) } - { // field ateapipb.Worker.Status + { // field ateapipb.WorkerAllocation.Allocated fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.WorkerStatus, + obj, oldObj *ateapipb.WorkerResources, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -6668,14 +6781,14 @@ func Validate_Worker( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_WorkerStatus(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_WorkerResources(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.Worker) *ateapipb.WorkerStatus { - return oldObj.Status + func(oldObj *ateapipb.WorkerAllocation) *ateapipb.WorkerResources { + return oldObj.Allocated }) - errs = append(errs, fn(fldPath.Child("status"), obj.Status, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("allocated"), obj.Allocated, oldVal, oldObj != nil)...) } return errs @@ -6901,47 +7014,46 @@ func Validate_WorkerAssignment( return errs } -// Validate_WorkerCapacity validates an instance of WorkerCapacity according +// Validate_WorkerResources validates an instance of WorkerResources according // to declarative validation rules in the API schema. -func Validate_WorkerCapacity( +func Validate_WorkerResources( ctx context.Context, op operation.Operation, fldPath *field.Path, - obj, oldObj *ateapipb.WorkerCapacity) (errs field.ErrorList) { + obj, oldObj *ateapipb.WorkerResources) (errs field.ErrorList) { - { // field ateapipb.WorkerCapacity.CpuMilli + { // field ateapipb.WorkerResources.Resources fn := func( fldPath *field.Path, - obj, oldObj *int64, + obj, oldObj *ateapipb.Resources, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { - if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + if ateDeepEqual(obj, oldObj) { return nil } } // call field-attached validations earlyReturn := false - if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { earlyReturn = true } if earlyReturn { return // do not proceed } - if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { - errs = append(errs, e...) - } + // call the type's validation function + errs = append(errs, Validate_Resources(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.WorkerCapacity) *int64 { - return &oldObj.CpuMilli + func(oldObj *ateapipb.WorkerResources) *ateapipb.Resources { + return oldObj.Resources }) - errs = append(errs, fn(fldPath.Child("cpu_milli"), &obj.CpuMilli, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("resources"), obj.Resources, oldVal, oldObj != nil)...) } - { // field ateapipb.WorkerCapacity.MemoryBytes + { // field ateapipb.WorkerResources.Actors fn := func( fldPath *field.Path, - obj, oldObj *int64, + obj, oldObj *int32, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -6963,10 +7075,10 @@ func Validate_WorkerCapacity( return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.WorkerCapacity) *int64 { - return &oldObj.MemoryBytes + func(oldObj *ateapipb.WorkerResources) *int32 { + return &oldObj.Actors }) - errs = append(errs, fn(fldPath.Child("memory_bytes"), &obj.MemoryBytes, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("actors"), &obj.Actors, oldVal, oldObj != nil)...) } return errs @@ -7013,10 +7125,10 @@ func Validate_WorkerStatus( errs = append(errs, fn(fldPath.Child("state"), &obj.State, oldVal, oldObj != nil)...) } - { // field ateapipb.WorkerStatus.Assignment + { // field ateapipb.WorkerStatus.Allocation fn := func( fldPath *field.Path, - obj, oldObj *ateapipb.ActorAssignment, + obj, oldObj *ateapipb.WorkerAllocation, oldValueCorrelated bool) (errs field.ErrorList) { // don't revalidate unchanged data if oldValueCorrelated && op.Type == operation.Update { @@ -7033,14 +7145,14 @@ func Validate_WorkerStatus( return // do not proceed } // call the type's validation function - errs = append(errs, Validate_ActorAssignment(ctx, op, fldPath, obj, oldObj)...) + errs = append(errs, Validate_WorkerAllocation(ctx, op, fldPath, obj, oldObj)...) return } oldVal := safe.Field(oldObj, - func(oldObj *ateapipb.WorkerStatus) *ateapipb.ActorAssignment { - return oldObj.Assignment + func(oldObj *ateapipb.WorkerStatus) *ateapipb.WorkerAllocation { + return oldObj.Allocation }) - errs = append(errs, fn(fldPath.Child("assignment"), obj.Assignment, oldVal, oldObj != nil)...) + errs = append(errs, fn(fldPath.Child("allocation"), obj.Allocation, oldVal, oldObj != nil)...) } return errs diff --git a/cmd/ateapi/internal/scheduling/metrics.go b/cmd/ateapi/internal/scheduling/metrics.go index 546b6d3cae..93f1c2a00b 100644 --- a/cmd/ateapi/internal/scheduling/metrics.go +++ b/cmd/ateapi/internal/scheduling/metrics.go @@ -75,7 +75,7 @@ func (s *scheduler) recordEligibleWorkers(ctx context.Context, matching []*ateap if _, ok := eligibleByPool[k]; !ok { eligibleByPool[k] = 0 } - if w.GetStatus().GetAssignment() == nil { + if s.HasRoom(w, constraints) { eligibleByPool[k]++ } } diff --git a/cmd/ateapi/internal/scheduling/scheduling.go b/cmd/ateapi/internal/scheduling/scheduling.go index 896d5ca1dd..ea45b4b945 100644 --- a/cmd/ateapi/internal/scheduling/scheduling.go +++ b/cmd/ateapi/internal/scheduling/scheduling.go @@ -22,6 +22,7 @@ import ( "math/rand" "slices" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" "go.opentelemetry.io/otel/metric" "k8s.io/apimachinery/pkg/labels" @@ -42,14 +43,9 @@ type Constraints struct { // to specific node VMs. RequiredNodes []string - // CPUMilli and MemoryBytes are the actor's declared resource limits, from - // the ActorTemplate. A worker is eligible only if its reported capacity is - // >= these. Zero means "unconstrained" for that dimension (the actor did not - // declare a limit), and a worker that reports zero capacity for a dimension - // is treated as unconstrained too, so placement is never blocked by missing - // data (matching the pre-capacity behavior). - CPUMilli int64 - MemoryBytes int64 + // Limits are the actor's declared resource limits, named as a Worker names + // the capacity it reports, so the two subtract. + Limits *ateapipb.Resources } // ErrNoCapacity is returned by Schedule when no free worker satisfies the @@ -62,8 +58,13 @@ type Scheduler interface { // Returns ErrNoCapacity when no free worker satisfies the requested constraints. Schedule(ctx context.Context, constraints Constraints) (*ateapipb.Worker, error) - // Applies reports whether worker satisfies constraints. + // Applies reports whether worker satisfies non-capacity constraints. Capacity + // is excluded so an existing assignment does not make its Worker ineligible. Applies(worker *ateapipb.Worker, constraints Constraints) bool + + // HasRoom reports whether worker's remaining capacity admits one more actor + // of this size, in every dimension. Independent of Applies. + HasRoom(worker *ateapipb.Worker, constraints Constraints) bool } // WorkerSource provides the whole fleet of workers. @@ -105,7 +106,6 @@ func (s *scheduler) Schedule(ctx context.Context, constraints Constraints) (*ate return nil, fmt.Errorf("while listing workers: %w", err) } - // Filter for candidate workers that are unassigned and meet all scheduling constraints matching := make([]*ateapipb.Worker, 0, len(workers)) var candidates []*ateapipb.Worker for _, worker := range workers { @@ -113,7 +113,7 @@ func (s *scheduler) Schedule(ctx context.Context, constraints Constraints) (*ate continue } matching = append(matching, worker) - if worker.GetStatus().GetAssignment() == nil { + if s.HasRoom(worker, constraints) { candidates = append(candidates, worker) } } @@ -145,17 +145,41 @@ func (s *scheduler) Applies(worker *ateapipb.Worker, constraints Constraints) bo return false } - // The worker must be able to contain the actor's declared limits. A zero - // constraint (actor declared no limit) or zero worker capacity (capacity - // unknown) is treated as unconstrained, so placement is never blocked by - // missing data. - capacity := worker.GetCapacity() - if constraints.CPUMilli > 0 && capacity.GetCpuMilli() > 0 && capacity.GetCpuMilli() < constraints.CPUMilli { + return len(constraints.RequiredNodes) == 0 || slices.Contains(constraints.RequiredNodes, worker.GetNodeName()) +} + +// HasRoom reports whether what the worker has left admits one more actor of +// this size. A dimension the worker does not report is unconstrained, so +// placement is never blocked by missing data. +// +// A worker whose recorded capacity or allocation will not parse is treated as +// having no room: it is the only answer that cannot overcommit a worker whose +// true occupancy is unreadable. +func (s *scheduler) HasRoom(worker *ateapipb.Worker, constraints Constraints) bool { + capacity := worker.GetStatus().GetAllocation().GetCapacity() + used := worker.GetStatus().GetAllocation().GetAllocated() + + // No per-actor size to compare: every assignment costs one, so a worker at + // its limit has no room however small the next actor is. + if used.GetActors() >= capacity.GetActors() { + return false + } + + want, err := resources.ParseQuantities(constraints.Limits) + if err != nil || len(want) == 0 { + return err == nil + } + free, err := resources.ParseQuantities(capacity.GetResources()) + if err != nil { return false } - if constraints.MemoryBytes > 0 && capacity.GetMemoryBytes() > 0 && capacity.GetMemoryBytes() < constraints.MemoryBytes { + if free == nil { + free = resources.Quantities{} + } + allocated, err := resources.ParseQuantities(used.GetResources()) + if err != nil { return false } - - return len(constraints.RequiredNodes) == 0 || slices.Contains(constraints.RequiredNodes, worker.GetNodeName()) + free.Sub(allocated) + return free.Covers(want) } diff --git a/cmd/ateapi/internal/scheduling/scheduling_test.go b/cmd/ateapi/internal/scheduling/scheduling_test.go index 9e17d8147c..c8a53a153b 100644 --- a/cmd/ateapi/internal/scheduling/scheduling_test.go +++ b/cmd/ateapi/internal/scheduling/scheduling_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" @@ -126,7 +127,7 @@ func TestSchedule(t *testing.T) { worker("w-small", "gvisor", "node-a", tierTwo, withCapacity(1000, 8<<30)), worker("w-big", "gvisor", "node-a", tierTwo, withCapacity(4000, 8<<30)), }, - constraints: Constraints{SandboxClass: "gvisor", CPUMilli: 2000}, + constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(2000, 0)}, wantPod: "w-big", }, { @@ -135,7 +136,7 @@ func TestSchedule(t *testing.T) { worker("w-small", "gvisor", "node-a", tierTwo, withCapacity(4000, 1<<30)), worker("w-big", "gvisor", "node-a", tierTwo, withCapacity(4000, 4<<30)), }, - constraints: Constraints{SandboxClass: "gvisor", MemoryBytes: 2 << 30}, + constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(0, 2<<30)}, wantPod: "w-big", }, { @@ -143,14 +144,25 @@ func TestSchedule(t *testing.T) { fleet: fleet{ worker("w-small", "gvisor", "node-a", tierTwo, withCapacity(1000, 1<<30)), }, - constraints: Constraints{SandboxClass: "gvisor", CPUMilli: 2000}, + constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(2000, 0)}, }, { - name: "zero worker capacity is treated as unconstrained", + // A Worker reports everything it has, so one that has reported no + // compute has none: an Actor that asks for some is not placed here. + name: "a worker that reported no compute takes no actor that needs some", fleet: fleet{ worker("w-unknown", "gvisor", "node-a", tierTwo), }, - constraints: Constraints{SandboxClass: "gvisor", CPUMilli: 2000, MemoryBytes: 2 << 30}, + constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(2000, 2<<30)}, + }, + { + // An Actor that declares nothing still fits: it asks for no + // dimension, so there is none the Worker must supply. + name: "a worker that reported no compute still takes an actor that needs none", + fleet: fleet{ + worker("w-unknown", "gvisor", "node-a", tierTwo), + }, + constraints: Constraints{SandboxClass: "gvisor"}, wantPod: "w-unknown", }, { @@ -166,6 +178,50 @@ func TestSchedule(t *testing.T) { fleet: fleet{}, constraints: Constraints{SandboxClass: "gvisor"}, }, + { + // A worker that has not said it can hold more admits one, so this is + // the behavior every worker has until an ateom reports otherwise. + name: "unset actor capacity admits one actor", + fleet: fleet{ + worker("w-busy", "gvisor", "node-a", tierTwo, assigned("demo", "other")), + }, + constraints: Constraints{SandboxClass: "gvisor"}, + }, + { + name: "a worker below its actor ceiling still has room", + fleet: fleet{ + worker("w-two", "gvisor", "node-a", tierTwo, withMaxActors(2), assigned("demo", "other")), + }, + constraints: Constraints{SandboxClass: "gvisor"}, + wantPod: "w-two", + }, + { + name: "a worker at its actor ceiling is full however small the actor", + fleet: fleet{ + worker("w-two", "gvisor", "node-a", tierTwo, withMaxActors(2), + assigned("demo", "a"), assigned("demo", "b")), + }, + constraints: Constraints{SandboxClass: "gvisor"}, + }, + { + // Placement is against what is left, not against the whole capacity: + // the resident actor already took half of it. + name: "capacity already allocated is not offered twice", + fleet: fleet{ + worker("w-half", "gvisor", "node-a", tierTwo, withCapacity(4000, 8<<30), withMaxActors(4), + assignedFor("demo", "other", resources.CPUMemory(3000, 4<<30))), + }, + constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(2000, 0)}, + }, + { + name: "what the residents left over is still placeable", + fleet: fleet{ + worker("w-half", "gvisor", "node-a", tierTwo, withCapacity(4000, 8<<30), withMaxActors(4), + assignedFor("demo", "other", resources.CPUMemory(3000, 4<<30))), + }, + constraints: Constraints{SandboxClass: "gvisor", Limits: resources.CPUMemory(1000, 4<<30)}, + wantPod: "w-half", + }, } for _, tc := range tests { @@ -287,9 +343,8 @@ func worker(pod, class, node string, lbls map[string]string, opts ...func(*ateap SandboxClass: class, NodeName: node, Labels: lbls, - Status: &ateapipb.WorkerStatus{ - State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, - }, + // A stored Worker always carries a ceiling; CreateWorker reifies one. + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}}}, } for _, opt := range opts { opt(w) @@ -304,17 +359,61 @@ func withState(state ateapipb.WorkerState) func(*ateapipb.Worker) { } func assigned(atespace, name string) func(*ateapipb.Worker) { + return assignedFor(atespace, name, nil) +} + +// assignedFor books an actor that took resources from the worker, so a test can +// place against what is left rather than against the whole capacity. Only the +// allocation total, which is all placement reads: the assignments themselves are +// their own records and never on the worker. +func assignedFor(atespace, name string, took *ateapipb.Resources) func(*ateapipb.Worker) { return func(w *ateapipb.Worker) { - w.Status.Assignment = &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: atespace, Name: name}, - ActorUid: atespace + "/" + name, + if w.Status == nil { + w.Status = &ateapipb.WorkerStatus{} } + allocated, err := resources.AddToAllocated(w.Status.Allocation.Allocated, + &ateapipb.ActorAssignment{ActorUid: atespace + "/" + name, Resources: took}, +1) + if err != nil { + panic(err) + } + w.Status.Allocation.Allocated = allocated + } +} + +func withMaxActors(n int32) func(*ateapipb.Worker) { + return func(w *ateapipb.Worker) { + if w.Status == nil { + w.Status = &ateapipb.WorkerStatus{} + } + if w.Status.Allocation.Capacity == nil { + w.Status.Allocation.Capacity = &ateapipb.WorkerResources{} + } + w.Status.Allocation.Capacity.Actors = n } } func withCapacity(cpuMilli, memBytes int64) func(*ateapipb.Worker) { return func(w *ateapipb.Worker) { - w.Capacity = &ateapipb.WorkerCapacity{CpuMilli: cpuMilli, MemoryBytes: memBytes} + if w.Status.Allocation.Capacity == nil { + w.Status.Allocation.Capacity = &ateapipb.WorkerResources{} + } + w.Status.Allocation.Capacity.Resources = resources.CPUMemory(cpuMilli, memBytes) + } +} + +// The two questions are separate because a caller re-validating a worker that +// already holds the actor must not be told the placement is illegal just +// because the actor it is asking about filled the worker up. +func TestAppliesIgnoresRoom(t *testing.T) { + full := worker("w-full", "gvisor", "node-a", nil, withMaxActors(1), assigned("demo", "resident")) + constraints := Constraints{SandboxClass: "gvisor"} + s := New(fleet{full}) + + if !s.Applies(full, constraints) { + t.Error("Applies() = false for a full but otherwise legal worker, want true") + } + if s.HasRoom(full, constraints) { + t.Error("HasRoom() = true for a worker at its actor ceiling, want false") } } diff --git a/cmd/ateapi/internal/store/atepg/atepg.go b/cmd/ateapi/internal/store/atepg/atepg.go index 914c8f00d0..0f593b5ad8 100644 --- a/cmd/ateapi/internal/store/atepg/atepg.go +++ b/cmd/ateapi/internal/store/atepg/atepg.go @@ -1456,6 +1456,309 @@ func (p *Persistence) DeleteWorker(ctx context.Context, name string, pre store.D }) } +// Worker assignments and status.allocated are updated in one transaction. + +// getWorkerForUpdate reads a Worker and holds its row until the caller's +// transaction commits, which is what serializes updates to its allocation. The +// caller supplies the transaction; this only takes the lock. +func getWorkerForUpdate(ctx context.Context, tx pgx.Tx, name string) (*ateapipb.Worker, error) { + var protoBytes []byte + err := tx.QueryRow(ctx, `SELECT proto FROM workers WHERE name = $1 FOR UPDATE`, name).Scan(&protoBytes) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, store.ErrNotFound + } + return nil, fmt.Errorf("locking worker %s: %w", name, err) + } + worker := &ateapipb.Worker{} + if err := proto.Unmarshal(protoBytes, worker); err != nil { + return nil, fmt.Errorf("unmarshaling worker: %w", err) + } + return worker, nil +} + +// saveWorker writes back a Worker whose allocation just moved, at the next +// version. Safe without a precondition only because the caller holds the row +// lock getWorkerForUpdate took. +func saveWorker(ctx context.Context, tx pgx.Tx, worker *ateapipb.Worker) error { + read := worker.GetMetadata() + worker.Metadata = newUpdateMetadata(read) + protoBytes, err := proto.Marshal(worker) + if err != nil { + return fmt.Errorf("marshaling worker: %w", err) + } + // Callers hold the row lock getWorkerForUpdate took, so this matches the + // row they read. It is stated anyway so a caller that skipped the lock + // fails loudly instead of overwriting a newer Worker, and so a row that is + // gone is an error rather than an update of nothing. + tag, err := tx.Exec(ctx, `UPDATE workers SET version = $1, proto = $2 WHERE name = $3 AND uid = $4 AND version = $5`, + worker.GetMetadata().GetVersion(), protoBytes, read.GetName(), read.GetUid(), read.GetVersion()) + if err != nil { + return fmt.Errorf("updating worker %s: %w", read.GetName(), err) + } + if tag.RowsAffected() != 1 { + return store.ErrVersionConflict + } + return nil +} + +func (p *Persistence) BindActorToWorker(ctx context.Context, workerName string, assignment *ateapipb.ActorAssignment, admit func(*ateapipb.Worker) error) error { + actorUID := assignment.GetActorUid() + if actorUID == "" { + return fmt.Errorf("binding an assignment with no actor_uid to worker %s", workerName) + } + // The store assigns identity. atespace is empty because Workers are + // global-scoped; the name is the Actor's UID, which is also the row key. + // This is the identity a first bind gets; a rebind keeps the recorded one. + assignment.Metadata = &ateapipb.ResourceMetadata{Name: actorUID} + setCreateMetadata(assignment.Metadata) + assignmentBytes, err := proto.Marshal(assignment) + if err != nil { + return fmt.Errorf("marshaling assignment: %w", err) + } + + _, err = p.writeAndAppendEvent(ctx, store.WorkerEventUpdated, func(ctx context.Context, tx pgx.Tx) (*ateapipb.Worker, error) { + worker, err := getWorkerForUpdate(ctx, tx, workerName) + if err != nil { + return nil, err + } + if worker.Status == nil { + worker.Status = &ateapipb.WorkerStatus{} + } + + // Insert first and let the conflict say whether the Actor was already + // bound. Checking with a read instead would miss a claim that commits + // after it, and both claims would believe they were first. + tag, err := tx.Exec(ctx, ` + INSERT INTO worker_assignments (actor_uid, worker_name, proto) + VALUES ($1, $2, $3) + ON CONFLICT (actor_uid) DO NOTHING`, + actorUID, workerName, assignmentBytes) + if err != nil { + return nil, fmt.Errorf("binding actor %s to worker %s: %w", actorUID, workerName, err) + } + if tag.RowsAffected() == 1 { + // A new binding needs room. The row lock holds the answer until + // commit, and a refusal rolls the insert back. + if admit != nil { + if err := admit(worker); err != nil { + return nil, err + } + } + allocated, err := resources.AddToAllocated(resources.Allocation(worker).Allocated, assignment, +1) + if err != nil { + return nil, err + } + resources.Allocation(worker).Allocated = allocated + if err := saveWorker(ctx, tx, worker); err != nil { + return nil, err + } + return worker, nil + } + + // Already bound. Only this path, the retried claim, pays for the read. + previous, previousWorker, err := getAssignmentRow(ctx, tx, actorUID) + if err != nil { + return nil, err + } + if previousWorker != workerName { + return nil, fmt.Errorf("actor %s is already hosted by worker %s", actorUID, previousWorker) + } + + // Subtract before adding: the Actor is already counted, and its + // declared size may have changed. + allocated, err := resources.AddToAllocated(resources.Allocation(worker).Allocated, previous, -1) + if err != nil { + return nil, err + } + resources.Allocation(worker).Allocated = allocated + + // Admit against the Worker without the old reservation. An + // ActorTemplate is mutable, so a replacement can be larger than what + // it replaces. + if admit != nil { + if err := admit(worker); err != nil { + return nil, err + } + } + + if allocated, err = resources.AddToAllocated(allocated, assignment, +1); err != nil { + return nil, err + } + resources.Allocation(worker).Allocated = allocated + + // A rebind updates the assignment already recorded, so re-stamping it + // as a create would make a retried claim look like a new subresource. + setUpdateMetadata(assignment.Metadata, previous.GetMetadata()) + rebindBytes, err := proto.Marshal(assignment) + if err != nil { + return nil, fmt.Errorf("marshaling rebound assignment: %w", err) + } + + // Guarded on worker_name so a claim that moved the Actor elsewhere is + // refused rather than overwritten. + rebind, err := tx.Exec(ctx, ` + UPDATE worker_assignments SET proto = $3 + WHERE actor_uid = $1 AND worker_name = $2`, + actorUID, workerName, rebindBytes) + if err != nil { + return nil, fmt.Errorf("rebinding actor %s on worker %s: %w", actorUID, workerName, err) + } + if rebind.RowsAffected() != 1 { + return nil, fmt.Errorf("%w: actor %s left worker %s while it was being rebound", store.ErrVersionConflict, actorUID, workerName) + } + if err := saveWorker(ctx, tx, worker); err != nil { + return nil, err + } + return worker, nil + }) + return err +} + +func (p *Persistence) ReleaseActorFromWorker(ctx context.Context, workerName string, actorUID string) (*ateapipb.Worker, error) { + var released *ateapipb.Worker + _, err := p.writeAndAppendEvent(ctx, store.WorkerEventUpdated, func(ctx context.Context, tx pgx.Tx) (*ateapipb.Worker, error) { + released = nil + worker, err := getWorkerForUpdate(ctx, tx, workerName) + if err != nil { + return nil, err + } + + var protoBytes []byte + err = tx.QueryRow(ctx, ` + DELETE FROM worker_assignments + WHERE actor_uid = $1 AND worker_name = $2 + RETURNING proto`, actorUID, workerName).Scan(&protoBytes) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil // nothing to release, and so nothing to announce + } + if err != nil { + return nil, fmt.Errorf("releasing actor %s from worker %s: %w", actorUID, workerName, err) + } + assignment := &ateapipb.ActorAssignment{} + if err := proto.Unmarshal(protoBytes, assignment); err != nil { + return nil, fmt.Errorf("unmarshaling released assignment: %w", err) + } + + if worker.Status == nil { + worker.Status = &ateapipb.WorkerStatus{} + } + allocated, err := resources.AddToAllocated(resources.Allocation(worker).Allocated, assignment, -1) + if err != nil { + return nil, err + } + resources.Allocation(worker).Allocated = allocated + if err := saveWorker(ctx, tx, worker); err != nil { + return nil, err + } + released = worker + return worker, nil + }) + if err != nil { + return nil, err + } + return released, nil +} + +// getAssignmentRow reads the assignment for actorUID and names the worker +// holding it. +func getAssignmentRow(ctx context.Context, q querier, actorUID string) (*ateapipb.ActorAssignment, string, error) { + var ( + protoBytes []byte + workerName string + ) + err := q.QueryRow(ctx, `SELECT proto, worker_name FROM worker_assignments WHERE actor_uid = $1`, actorUID). + Scan(&protoBytes, &workerName) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, "", store.ErrNotFound + } + return nil, "", fmt.Errorf("getting assignment for actor %s: %w", actorUID, err) + } + assignment := &ateapipb.ActorAssignment{} + if err := proto.Unmarshal(protoBytes, assignment); err != nil { + return nil, "", fmt.Errorf("unmarshaling assignment: %w", err) + } + return assignment, workerName, nil +} + +func (p *Persistence) GetWorkerAssignment(ctx context.Context, workerName, actorUID string) (*ateapipb.ActorAssignment, error) { + assignment, holder, err := getAssignmentRow(ctx, p.pool, actorUID) + if err != nil { + return nil, err + } + if holder != workerName { + return nil, store.ErrNotFound + } + return assignment, nil +} + +func (p *Persistence) ListWorkerAssignments(ctx context.Context, workerName string, opts store.ListOptions) (store.ListResponse[*ateapipb.ActorAssignment], error) { + opts, err := store.NormalizeListOptions(opts) + if err != nil { + return store.ListResponse[*ateapipb.ActorAssignment]{}, err + } + pageSize := opts.PageSize + // The token is scoped to the Worker, so one cannot be replayed against + // another Worker's assignments. + token, err := decodePageToken(opts.PageToken, kindWorkerAssign, workerName, 1) + if err != nil { + return store.ListResponse[*ateapipb.ActorAssignment]{}, err + } + var last *string + if len(token.Last) > 0 { + last = &token.Last[0] + } + + rows, err := p.pool.Query(ctx, ` + SELECT actor_uid, proto FROM worker_assignments + WHERE worker_name = $1 AND ($2::text IS NULL OR actor_uid > $2) + ORDER BY actor_uid + LIMIT $3`, workerName, last, int64(pageSize)+1) + if err != nil { + return store.ListResponse[*ateapipb.ActorAssignment]{}, fmt.Errorf("listing assignments of worker %s: %w", workerName, err) + } + defer rows.Close() + + var uids []string + var result []*ateapipb.ActorAssignment + for rows.Next() { + var actorUID string + var protoBytes []byte + if err := rows.Scan(&actorUID, &protoBytes); err != nil { + return store.ListResponse[*ateapipb.ActorAssignment]{}, fmt.Errorf("scanning assignment of worker %s: %w", workerName, err) + } + assignment := &ateapipb.ActorAssignment{} + if err := proto.Unmarshal(protoBytes, assignment); err != nil { + return store.ListResponse[*ateapipb.ActorAssignment]{}, fmt.Errorf("unmarshaling assignment: %w", err) + } + result = append(result, assignment) + uids = append(uids, actorUID) + } + if err := rows.Err(); err != nil { + return store.ListResponse[*ateapipb.ActorAssignment]{}, fmt.Errorf("listing assignments of worker %s: %w", workerName, err) + } + + var nextToken string + if len(result) > int(pageSize) { + result = result[:pageSize] + nextToken = encodePageToken(kindWorkerAssign, workerName, []string{uids[pageSize-1]}) + } + return store.ListResponse[*ateapipb.ActorAssignment]{Items: result, NextPageToken: nextToken}, nil +} + +func (p *Persistence) FindWorkerHostingActor(ctx context.Context, actorUID string) (string, error) { + var workerName string + err := p.pool.QueryRow(ctx, `SELECT worker_name FROM worker_assignments WHERE actor_uid = $1`, actorUID).Scan(&workerName) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return "", store.ErrNotFound + } + return "", fmt.Errorf("finding the worker hosting actor %s: %w", actorUID, err) + } + return workerName, nil +} + func (p *Persistence) ListWorkers(ctx context.Context, opts store.ListOptions) (store.ListResponse[*ateapipb.Worker], error) { opts, err := store.NormalizeListOptions(opts) if err != nil { diff --git a/cmd/ateapi/internal/store/atepg/atepg_test.go b/cmd/ateapi/internal/store/atepg/atepg_test.go index bd9705f36e..62e6b38247 100644 --- a/cmd/ateapi/internal/store/atepg/atepg_test.go +++ b/cmd/ateapi/internal/store/atepg/atepg_test.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "io/fs" "os" "strings" "sync" @@ -144,12 +145,18 @@ func TestMigrationsConcurrentStartup(t *testing.T) { } } + // Every migration applied once and no more: two racing starts must not each + // record the same version. + want, err := fs.Glob(migrationFiles, "migrations/*.sql") + if err != nil { + t.Fatalf("listing migrations: %v", err) + } var applied int if err := pool.QueryRow(ctx, `SELECT count(*) FROM "concurrent-startup".schema_migrations WHERE version_id > 0 AND is_applied`).Scan(&applied); err != nil { t.Fatalf("reading applied migrations: %v", err) } - if applied != 1 { - t.Fatalf("applied migration rows = %d, want 1", applied) + if applied != len(want) { + t.Fatalf("applied migration rows = %d, want %d", applied, len(want)) } } @@ -452,7 +459,7 @@ func appliedMigrationVersions(t *testing.T, pool *pgxpool.Pool) []int64 { // state, so the statement lives here rather than on Persistence. func clearAll(t *testing.T, p *Persistence) { t.Helper() - if _, err := p.pool.Exec(context.Background(), `TRUNCATE atespaces, actors, actor_egress_policies, actor_templates, actor_snapshots, actor_snapshot_tags, workers, leases, worker_outbox, worker_outbox_trim`); err != nil { + if _, err := p.pool.Exec(context.Background(), `TRUNCATE atespaces, actors, actor_egress_policies, actor_templates, actor_snapshots, actor_snapshot_tags, workers, worker_assignments, leases, worker_outbox, worker_outbox_trim`); err != nil { t.Fatalf("truncating tables: %v", err) } } @@ -837,3 +844,83 @@ func TestAcquireLease_ConcurrentTakeover(t *testing.T) { lease.Close() } } + +// TestSaveWorker_RejectsAStaleWrite proves the precondition saveWorker states +// on top of the row lock its callers hold: a Worker read before someone else +// wrote it cannot overwrite that write. +func TestSaveWorker_RejectsAStaleWrite(t *testing.T) { + requirePool(t) + ctx := context.Background() + + p, err := Connect(ctx, containerDSN, "public") + if err != nil { + t.Fatalf("Connect failed: %v", err) + } + defer p.pool.Close() + defer p.Close() + clearAll(t, p) + + created, err := p.CreateWorker(ctx, &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: "stale-write-worker"}, + WorkerNamespace: "ns", + WorkerPool: "pool", + WorkerPod: "pod", + }) + if err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + + // Move the stored Worker on, so the copy above is a version behind. + if _, err := p.UpdateWorker(ctx, created.GetMetadata().GetName(), store.PreconditionFrom(created), func(toUpdate *ateapipb.Worker) error { + toUpdate.Ip = "10.0.0.1" + return nil + }); err != nil { + t.Fatalf("UpdateWorker failed: %v", err) + } + + tx, err := p.pool.Begin(ctx) + if err != nil { + t.Fatalf("Begin failed: %v", err) + } + defer tx.Rollback(ctx) + if err := saveWorker(ctx, tx, created); !errors.Is(err, store.ErrVersionConflict) { + t.Errorf("saveWorker() with a stale Worker = %v, want ErrVersionConflict", err) + } +} + +// TestSaveWorker_RejectsAVanishedWorker keeps a deleted row from being an +// update of nothing. +func TestSaveWorker_RejectsAVanishedWorker(t *testing.T) { + requirePool(t) + ctx := context.Background() + + p, err := Connect(ctx, containerDSN, "public") + if err != nil { + t.Fatalf("Connect failed: %v", err) + } + defer p.pool.Close() + defer p.Close() + clearAll(t, p) + + created, err := p.CreateWorker(ctx, &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: "vanished-worker"}, + WorkerNamespace: "ns", + WorkerPool: "pool", + WorkerPod: "pod", + }) + if err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + if _, err := p.DeleteWorker(ctx, created.GetMetadata().GetName(), store.DeletePreconditions{}); err != nil { + t.Fatalf("DeleteWorker failed: %v", err) + } + + tx, err := p.pool.Begin(ctx) + if err != nil { + t.Fatalf("Begin failed: %v", err) + } + defer tx.Rollback(ctx) + if err := saveWorker(ctx, tx, created); !errors.Is(err, store.ErrVersionConflict) { + t.Errorf("saveWorker() on a deleted Worker = %v, want ErrVersionConflict", err) + } +} diff --git a/cmd/ateapi/internal/store/atepg/migrations/000001_initial.sql b/cmd/ateapi/internal/store/atepg/migrations/000001_initial.sql index a5b485550f..e744f972c9 100644 --- a/cmd/ateapi/internal/store/atepg/migrations/000001_initial.sql +++ b/cmd/ateapi/internal/store/atepg/migrations/000001_initial.sql @@ -89,6 +89,20 @@ CREATE TABLE workers ( proto bytea NOT NULL ); +-- One row per Actor, keyed by Actor UID because an Actor has at most one +-- Worker. Kept separate from workers so Worker reads, writes, and watch events +-- do not grow with occupancy. The primary key finds an Actor's Worker; +-- worker_name lists a Worker's Actors. +CREATE TABLE worker_assignments ( + actor_uid text PRIMARY KEY, + worker_name text NOT NULL + REFERENCES workers(name) ON DELETE CASCADE, + proto bytea NOT NULL +); + +CREATE INDEX worker_assignments_worker_idx + ON worker_assignments (worker_name); + -- Transactional outbox backing WatchWorkers. -- -- 1. Ordering (xid): writeAndAppendEvent guarantees exactly one row per tx, diff --git a/cmd/ateapi/internal/store/atepg/pagetoken.go b/cmd/ateapi/internal/store/atepg/pagetoken.go index 4a316390f5..872582a03a 100644 --- a/cmd/ateapi/internal/store/atepg/pagetoken.go +++ b/cmd/ateapi/internal/store/atepg/pagetoken.go @@ -36,6 +36,7 @@ const ( kindActorTemplate resourceKind = "actor-template" kindSnapshot resourceKind = "snapshot" kindWorker resourceKind = "worker" + kindWorkerAssign resourceKind = "worker-assignment" ) // pageToken is PostgreSQL's opaque keyset page token. It carries no database diff --git a/cmd/ateapi/internal/store/store.go b/cmd/ateapi/internal/store/store.go index 39411087cb..816f32d36d 100644 --- a/cmd/ateapi/internal/store/store.go +++ b/cmd/ateapi/internal/store/store.go @@ -222,11 +222,43 @@ type Interface interface { // exhausted, or the mutate's error verbatim otherwise. UpdateWorker(ctx context.Context, name string, precondition Precondition, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) - // Removes a worker by name and returns the deleted resource. Returns - // ErrNotFound if missing, or ErrUIDConflict/ErrVersionConflict if pre does - // not describe the worker the caller observed. + // Removes a worker by name, along with every assignment it holds, and + // returns the deleted resource. Returns ErrNotFound if missing, or + // ErrUIDConflict/ErrVersionConflict if pre does not describe the worker the + // caller observed. DeleteWorker(ctx context.Context, name string, pre DeletePreconditions) (*ateapipb.Worker, error) + // Assignments and Worker allocation are updated atomically. + + // BindActorToWorker assigns an Actor and updates the Worker's allocation. + // Rebinding the same Actor replaces its assignment. + // + // admit decides whether the Worker will take the Actor, and runs against the + // Worker as it stands with its row locked, so the answer cannot go stale + // between the check and the bind. Returning an error from it refuses the + // bind and is returned unchanged. It is consulted only for a new binding: an + // Actor already on this Worker is already counted against it. + // + // ErrNotFound if the Worker is gone. + BindActorToWorker(ctx context.Context, workerName string, assignment *ateapipb.ActorAssignment, admit func(*ateapipb.Worker) error) error + + // ReleaseActorFromWorker removes an assignment and updates allocation, + // returning the Worker as it now stands so the caller can feed the + // watch-fed cache, which until then reports it full. It returns nil if the + // assignment was already absent. + ReleaseActorFromWorker(ctx context.Context, workerName string, actorUID string) (*ateapipb.Worker, error) + + // GetWorkerAssignment returns a Worker's assignment for actorUID, or + // ErrNotFound when the Worker is not hosting that Actor. + GetWorkerAssignment(ctx context.Context, workerName, actorUID string) (*ateapipb.ActorAssignment, error) + + // ListWorkerAssignments returns a page of the Actors a Worker hosts. + ListWorkerAssignments(ctx context.Context, workerName string, opts ListOptions) (ListResponse[*ateapipb.ActorAssignment], error) + + // FindWorkerHostingActor names the Worker holding an assignment for + // actorUID, or ErrNotFound if none does. + FindWorkerHostingActor(ctx context.Context, actorUID string) (string, error) + // WatchWorkers returns an active subscription to track worker state changes. // The watch's Events channel is closed when the caller calls Close, the // context is cancelled, or the underlying notification system is lost. diff --git a/cmd/ateapi/internal/store/storecontract/contract.go b/cmd/ateapi/internal/store/storecontract/contract.go index 9285dba2d3..ad9eb3f0e6 100644 --- a/cmd/ateapi/internal/store/storecontract/contract.go +++ b/cmd/ateapi/internal/store/storecontract/contract.go @@ -20,7 +20,8 @@ import ( "context" "errors" "fmt" - "strings" + "math/rand/v2" + "slices" "sync" "testing" "time" @@ -110,8 +111,11 @@ func newTestWorker(name, pod string) *ateapipb.Worker { WorkerPool: "pool-1", WorkerPod: pod, WorkerPodUid: testWorkerPodUID, - Capacity: &ateapipb.WorkerCapacity{CpuMilli: 2000, MemoryBytes: 4 << 30}, - Status: &ateapipb.WorkerStatus{}, + Status: &ateapipb.WorkerStatus{ + Allocation: &ateapipb.WorkerAllocation{ + Capacity: &ateapipb.WorkerResources{Resources: resources.CPUMemory(2000, 4<<30)}, + }, + }, } } @@ -186,6 +190,7 @@ func RunContractTests(t *testing.T, setup func(t *testing.T) store.Interface) { runActorContractTests(t, setup) runEgressPolicyContractTests(t, setup) runWorkerContractTests(t, setup) + runWorkerAssignmentContractTests(t, setup) runAtespaceContractTests(t, setup) runActorTemplateContractTests(t, setup) runActorSnapshotContractTests(t, setup) @@ -1275,12 +1280,8 @@ func runWorkerContractTests(t *testing.T, setup func(t *testing.T) store.Interfa } defer watch.Close() - assignment := &ateapipb.ActorAssignment{ - ActorTemplateRef: &ateapipb.ObjectRef{Atespace: "default", Name: "test-template"}, - Actor: &ateapipb.ObjectRef{Name: "session-1"}, - } updated, err := s.UpdateWorker(ctx, testWorkerName, store.PreconditionFrom(created), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.Assignment = assignment + toUpdate.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING return nil }) if err != nil { @@ -1299,7 +1300,7 @@ func runWorkerContractTests(t *testing.T, setup func(t *testing.T) store.Interfa } want := proto.Clone(worker).(*ateapipb.Worker) - want.Status.Assignment = assignment + want.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING want.Metadata.Version = 2 if diff := cmp.Diff(want, got, protocmp.Transform(), ignoreUID, ignoreTimestamps); diff != "" { t.Errorf("UpdateWorker yielded unexpected state in DB (-want +got):\n%s", diff) @@ -1414,14 +1415,14 @@ func runWorkerContractTests(t *testing.T, setup func(t *testing.T) store.Interfa t.Fatalf("GetWorker failed: %v", err) } if _, err := s.UpdateWorker(ctx, testWorkerName, store.PreconditionFrom(observed), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.Assignment = &ateapipb.ActorAssignment{Actor: &ateapipb.ObjectRef{Name: "session-1"}} + toUpdate.Status.State = ateapipb.WorkerState_WORKER_STATE_DRAINING return nil }); err != nil { t.Fatalf("UpdateWorker failed: %v", err) } _, err = s.UpdateWorker(ctx, testWorkerName, store.PreconditionFrom(observed), func(toUpdate *ateapipb.Worker) error { - toUpdate.Status.Assignment = &ateapipb.ActorAssignment{Actor: &ateapipb.ObjectRef{Name: "session-2"}} + toUpdate.Status.State = ateapipb.WorkerState_WORKER_STATE_ACTIVE return nil }) if !errors.Is(err, store.ErrVersionConflict) { @@ -1485,14 +1486,15 @@ func runWorkerContractTests(t *testing.T, setup func(t *testing.T) store.Interfa wg.Add(1) go func() { defer wg.Done() + // Claimants contend over the allocation total, which is what a + // claim actually moves on the Worker record now that the + // assignments themselves are separate records. CpuMilli carries + // the claimant's number so the winner is identifiable. _, err := s.UpdateWorker(ctx, testWorkerName, store.PreconditionFrom(created), func(toUpdate *ateapipb.Worker) error { - if toUpdate.GetStatus().GetAssignment() != nil { + if toUpdate.GetStatus().GetAllocation().GetAllocated().GetActors() > 0 { return errTaken } - toUpdate.Status.Assignment = &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: fmt.Sprintf("actor-%d", i)}, - ActorUid: fmt.Sprintf("uid-%d", i), - } + toUpdate.Status.Allocation.Allocated = &ateapipb.WorkerResources{Actors: 1, Resources: resources.CPUMemory(int64(i)+1, 0)} return nil }) switch { @@ -1520,8 +1522,8 @@ func runWorkerContractTests(t *testing.T, setup func(t *testing.T) store.Interfa if err != nil { t.Fatalf("GetWorker failed: %v", err) } - if uid := got.GetStatus().GetAssignment().GetActorUid(); !strings.HasPrefix(uid, "uid-") { - t.Errorf("stored assignment names %q, want one of the claimants", uid) + if n := got.GetStatus().GetAllocation().GetAllocated().GetActors(); n < 1 || n > int32(claimants) { + t.Errorf("stored claim names %d, want one of the claimants", n) } // One winning write on top of the create, and no partial ones. if got.GetMetadata().GetVersion() != 2 { @@ -1706,6 +1708,673 @@ func runWorkerContractTests(t *testing.T, setup func(t *testing.T) store.Interfa }) } +// runWorkerAssignmentContractTests covers the records that say which Actors a +// Worker hosts, and the allocation total that has to move with them. +func runWorkerAssignmentContractTests(t *testing.T, setup func(t *testing.T) store.Interface) { + t.Helper() + + // newTestAssignment describes one Actor's placement, sized so that a total + // over several of them is unambiguous about which were counted. + newTestAssignment := func(actorUID string, cpuMilli, memoryBytes int64) *ateapipb.ActorAssignment { + return &ateapipb.ActorAssignment{ + ActorTemplateRef: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "test-template"}, + Actor: &ateapipb.ObjectRef{Atespace: testAtespace, Name: "actor-" + actorUID}, + ActorUid: actorUID, + Resources: resources.CPUMemory(cpuMilli, memoryBytes), + } + } + + // bind binds one assignment with no admission check, for the tests that care + // about the outcome rather than about admission. + bind := func(t *testing.T, s store.Interface, workerName string, assignment *ateapipb.ActorAssignment) { + t.Helper() + ctx := context.Background() + if err := s.BindActorToWorker(ctx, workerName, assignment, nil); err != nil { + t.Fatalf("BindActorToWorker(%s) failed: %v", assignment.GetActorUid(), err) + } + } + + t.Run("BindActorToWorker_AddsAssignmentAndAllocation", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + watch, err := s.WatchWorkers(ctx) + if err != nil { + t.Fatalf("WatchWorkers failed: %v", err) + } + defer watch.Close() + + assignment := newTestAssignment("uid-1", 500, 1<<20) + bind(t, s, testWorkerName, assignment) + + got, err := s.GetWorkerAssignment(ctx, testWorkerName, "uid-1") + if err != nil { + t.Fatalf("GetWorkerAssignment failed: %v", err) + } + if diff := cmp.Diff(assignment, got, protocmp.Transform()); diff != "" { + t.Errorf("stored assignment mismatch (-want +got):\n%s", diff) + } + + worker, err := s.GetWorker(ctx, testWorkerName) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + want := &ateapipb.WorkerResources{Actors: 1, Resources: resources.CPUMemory(500, 1<<20)} + if diff := cmp.Diff(want, worker.GetStatus().GetAllocation().GetAllocated(), protocmp.Transform()); diff != "" { + t.Errorf("allocated mismatch (-want +got):\n%s", diff) + } + if worker.GetMetadata().GetVersion() != 2 { + t.Errorf("worker version = %d, want 2: a bind advances the worker", worker.GetMetadata().GetVersion()) + } + + event := receiveEvent(t, watch.Events) + if event.Type != store.WorkerEventUpdated { + t.Errorf("expected WorkerEventUpdated, got %v", event.Type) + } + if diff := cmp.Diff(want, event.Worker.GetStatus().GetAllocation().GetAllocated(), protocmp.Transform()); diff != "" { + t.Errorf("event allocated mismatch (-want +got):\n%s", diff) + } + }) + + t.Run("BindActorToWorker_ReplacesSameActor", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + // A retried claim binds the same Actor twice. It must be counted once, + // at its latest size, or the Worker leaks capacity it never gave out. + bind(t, s, testWorkerName, newTestAssignment("uid-1", 500, 1<<20)) + bind(t, s, testWorkerName, newTestAssignment("uid-1", 250, 1<<21)) + + worker, err := s.GetWorker(ctx, testWorkerName) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + want := &ateapipb.WorkerResources{Actors: 1, Resources: resources.CPUMemory(250, 1<<21)} + if diff := cmp.Diff(want, worker.GetStatus().GetAllocation().GetAllocated(), protocmp.Transform()); diff != "" { + t.Errorf("allocated mismatch after rebinding the same actor (-want +got):\n%s", diff) + } + assignmentsPage, err := s.ListWorkerAssignments(ctx, testWorkerName, store.ListOptions{}) + if err != nil { + t.Fatalf("ListWorkerAssignments failed: %v", err) + } + assignments := assignmentsPage.Items + if len(assignments) != 1 { + t.Errorf("worker holds %d assignments, want 1", len(assignments)) + } + }) + + t.Run("BindActorToWorker_RebindKeepsAssignmentIdentity", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + bind(t, s, testWorkerName, newTestAssignment("uid-1", 500, 1<<20)) + first, err := s.GetWorkerAssignment(ctx, testWorkerName, "uid-1") + if err != nil { + t.Fatalf("GetWorkerAssignment failed: %v", err) + } + + // A retried claim rebinds the same Actor, which updates the assignment + // already recorded. A reader watching uid or create_time must not see a + // different subresource. + bind(t, s, testWorkerName, newTestAssignment("uid-1", 250, 1<<21)) + again, err := s.GetWorkerAssignment(ctx, testWorkerName, "uid-1") + if err != nil { + t.Fatalf("GetWorkerAssignment after rebind failed: %v", err) + } + + if got, want := again.GetMetadata().GetUid(), first.GetMetadata().GetUid(); got != want { + t.Errorf("rebind changed the assignment uid to %q, want %q", got, want) + } + if diff := cmp.Diff(first.GetMetadata().GetCreateTime(), again.GetMetadata().GetCreateTime(), protocmp.Transform()); diff != "" { + t.Errorf("rebind moved create_time (-want +got):\n%s", diff) + } + if got, want := again.GetMetadata().GetVersion(), first.GetMetadata().GetVersion()+1; got != want { + t.Errorf("rebind left version at %d, want %d: an update advances it", got, want) + } + }) + + t.Run("BindActorToWorker_RefusedAdmissionLeavesNothingBehind", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + + full := errors.New("worker is full") + err := s.BindActorToWorker(ctx, testWorkerName, newTestAssignment("uid-1", 500, 0), + func(*ateapipb.Worker) error { return full }) + if !errors.Is(err, full) { + t.Errorf("BindActorToWorker with a refusing admit = %v, want %v", err, full) + } + // The bind speculates the row in before it asks, so a refusal has to + // take it back out. + if _, err := s.GetWorkerAssignment(ctx, testWorkerName, "uid-1"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("a refused bind left an assignment behind: %v", err) + } + worker, err := s.GetWorker(ctx, testWorkerName) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + if got := worker.GetStatus().GetAllocation().GetAllocated().GetActors(); got != 0 { + t.Errorf("a refused bind left %d actors allocated, want 0", got) + } + }) + + // What the caller needs from admission: it runs against the Worker as the + // bind will write it, so concurrent binds cannot both find room for the last + // place. Nothing outside the store can offer that -- a check made before the + // call is stale by the time it commits. + // An ActorTemplate is mutable, so a retried claim can come back bigger than + // the one already booked. Admission has to run on the replacement, judged + // against the Worker with the old reservation taken off. + t.Run("BindActorToWorker_AdmitsAReplacementThatGrew", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + bind(t, s, testWorkerName, newTestAssignment("uid-1", 500, 0)) + + var admitted []*ateapipb.WorkerResources + full := errors.New("worker is full") + err := s.BindActorToWorker(ctx, testWorkerName, newTestAssignment("uid-1", 5000, 0), + func(fresh *ateapipb.Worker) error { + admitted = append(admitted, fresh.GetStatus().GetAllocation().GetAllocated()) + return full + }) + if !errors.Is(err, full) { + t.Fatalf("rebinding a grown assignment = %v, want the admit refusal %v", err, full) + } + if len(admitted) != 1 { + t.Fatalf("admit ran %d times on a replacement, want once", len(admitted)) + } + // Judged without the old reservation: otherwise the Actor is counted + // twice and a replacement that merely stayed the same size is refused. + if got := admitted[0].GetActors(); got != 0 { + t.Errorf("admit saw %d actors allocated, want 0: the previous reservation should be off", got) + } + + // A refusal leaves the original booking exactly as it was. + worker, err := s.GetWorker(ctx, testWorkerName) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + want := &ateapipb.WorkerResources{Actors: 1, Resources: resources.CPUMemory(500, 0)} + if diff := cmp.Diff(want, worker.GetStatus().GetAllocation().GetAllocated(), protocmp.Transform()); diff != "" { + t.Errorf("a refused replacement changed the allocation (-want +got):\n%s", diff) + } + }) + + t.Run("BindActorToWorker_AdmitSeesConcurrentBinds", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + + const claims = 16 + full := errors.New("worker is full") + roomForOne := func(w *ateapipb.Worker) error { + if w.GetStatus().GetAllocation().GetAllocated().GetActors() >= 1 { + return full + } + return nil + } + + var wg sync.WaitGroup + won := make([]bool, claims) + for i := range claims { + wg.Go(func() { + err := s.BindActorToWorker(ctx, testWorkerName, + newTestAssignment(fmt.Sprintf("uid-%d", i), 0, 0), roomForOne) + won[i] = err == nil + }) + } + wg.Wait() + + var winners int + for _, w := range won { + if w { + winners++ + } + } + if winners != 1 { + t.Errorf("%d of %d concurrent binds were admitted, want exactly 1", winners, claims) + } + worker, err := s.GetWorker(ctx, testWorkerName) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + if got := worker.GetStatus().GetAllocation().GetAllocated().GetActors(); got != 1 { + t.Errorf("worker allocation counts %d actors, want 1", got) + } + }) + + // An Actor belongs to one Worker. A second Worker claiming it must be + // refused rather than quietly taking the row, or the first goes on counting + // an Actor it does not host and never gets that capacity back. + t.Run("BindActorToWorker_ActorAlreadyOnAnotherWorker", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + for _, name := range []string{testWorkerName, otherTestWorkerName} { + if _, err := s.CreateWorker(ctx, newTestWorker(name, "pod-"+name)); err != nil { + t.Fatalf("CreateWorker(%s) failed: %v", name, err) + } + } + bind(t, s, testWorkerName, newTestAssignment("uid-1", 500, 1<<20)) + + other, err := s.GetWorker(ctx, otherTestWorkerName) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + if err := s.BindActorToWorker(ctx, otherTestWorkerName, newTestAssignment("uid-1", 500, 1<<20), nil); err == nil { + t.Fatal("BindActorToWorker onto a second worker succeeded, want an error") + } + + // The refusal has to leave both Workers exactly as they were: the first + // still hosting the Actor and counting it, the second counting nothing. + holder, err := s.FindWorkerHostingActor(ctx, "uid-1") + if err != nil { + t.Fatalf("FindWorkerHostingActor failed: %v", err) + } + if holder != testWorkerName { + t.Errorf("actor moved to worker %q, want it left on %q", holder, testWorkerName) + } + first, err := s.GetWorker(ctx, testWorkerName) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + want := &ateapipb.WorkerResources{Actors: 1, Resources: resources.CPUMemory(500, 1<<20)} + if diff := cmp.Diff(want, first.GetStatus().GetAllocation().GetAllocated(), protocmp.Transform()); diff != "" { + t.Errorf("first worker's allocation changed (-want +got):\n%s", diff) + } + refused, err := s.GetWorker(ctx, otherTestWorkerName) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + if got := refused.GetStatus().GetAllocation().GetAllocated().GetActors(); got != 0 { + t.Errorf("refused worker counts %d actors, want 0", got) + } + if got, wantVersion := refused.GetMetadata().GetVersion(), other.GetMetadata().GetVersion(); got != wantVersion { + t.Errorf("refused worker moved to version %d, want %d unchanged", got, wantVersion) + } + }) + + // The same rule under contention, which is where it is actually decided. + // Two replicas resuming one Actor pick different Workers and claim at the + // same moment; a backend that decides on a read taken before the competing + // write exists lets both through, and the loser is left counting an Actor + // whose row went elsewhere. + t.Run("BindActorToWorker_ConcurrentClaimsOfOneActor", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + names := []string{testWorkerName, otherTestWorkerName} + versions := make([]int64, len(names)) + for i, name := range names { + created, err := s.CreateWorker(ctx, newTestWorker(name, "pod-"+name)) + if err != nil { + t.Fatalf("CreateWorker(%s) failed: %v", name, err) + } + versions[i] = created.GetMetadata().GetVersion() + } + + var start sync.WaitGroup + start.Add(1) + var wg sync.WaitGroup + won := make([]bool, len(names)) + for i, name := range names { + wg.Add(1) + go func() { + defer wg.Done() + start.Wait() + won[i] = s.BindActorToWorker(ctx, name, newTestAssignment("uid-1", 500, 1<<20), nil) == nil + }() + } + start.Done() + wg.Wait() + + if winners := len(slices.DeleteFunc(slices.Clone(won), func(w bool) bool { return !w })); winners != 1 { + t.Errorf("%d of %d claims won the actor, want exactly 1", winners, len(names)) + } + // Whoever won, no Worker may count an Actor it does not hold. + for _, name := range names { + worker, err := s.GetWorker(ctx, name) + if err != nil { + t.Fatalf("GetWorker(%s) failed: %v", name, err) + } + assignmentsPage, err := s.ListWorkerAssignments(ctx, name, store.ListOptions{}) + if err != nil { + t.Fatalf("ListWorkerAssignments(%s) failed: %v", name, err) + } + assignments := assignmentsPage.Items + want, err := resources.SumAllocated(assignments) + if err != nil { + t.Fatalf("%s: summing assignments: %v", name, err) + } + // A Worker that never took anything has no allocation at all, while + // a sum over no assignments is a zeroed message; the two say the + // same thing, so compare what they mean rather than the messages. + got := worker.GetStatus().GetAllocation().GetAllocated() + if got.GetActors() != want.GetActors() || !proto.Equal(got.GetResources(), want.GetResources()) { + t.Errorf("%s: allocation %v disagrees with the %d assignments it holds (%v)", + name, got, len(assignments), want) + } + } + }) + + t.Run("BindActorToWorker_WorkerNotFound", func(t *testing.T) { + s := setup(t) + err := s.BindActorToWorker(context.Background(), "no-such-worker", newTestAssignment("uid-1", 0, 0), nil) + if !errors.Is(err, store.ErrNotFound) { + t.Errorf("BindActorToWorker on a missing worker = %v, want ErrNotFound", err) + } + }) + + t.Run("ReleaseActorFromWorker", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + bind(t, s, testWorkerName, newTestAssignment("uid-1", 500, 1<<20)) + bind(t, s, testWorkerName, newTestAssignment("uid-2", 250, 1<<21)) + + released, err := s.ReleaseActorFromWorker(ctx, testWorkerName, "uid-1") + if err != nil { + t.Fatalf("ReleaseActorFromWorker failed: %v", err) + } + if released == nil { + t.Fatal("ReleaseActorFromWorker reported nothing to release") + } + // The returned Worker is what the caller feeds the cache, so it has to + // be the post-release state, not the copy that went in. + wantReleased := &ateapipb.WorkerResources{Actors: 1, Resources: resources.CPUMemory(250, 1<<21)} + if diff := cmp.Diff(wantReleased, released.GetStatus().GetAllocation().GetAllocated(), protocmp.Transform()); diff != "" { + t.Errorf("returned worker's allocated mismatch (-want +got):\n%s", diff) + } + + // Only the released Actor goes; the Worker's others are untouched. + if _, err := s.GetWorkerAssignment(ctx, testWorkerName, "uid-1"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("released assignment is still readable: %v", err) + } + if _, err := s.GetWorkerAssignment(ctx, testWorkerName, "uid-2"); err != nil { + t.Errorf("releasing one actor disturbed another: %v", err) + } + worker, err := s.GetWorker(ctx, testWorkerName) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + want := &ateapipb.WorkerResources{Actors: 1, Resources: resources.CPUMemory(250, 1<<21)} + if diff := cmp.Diff(want, worker.GetStatus().GetAllocation().GetAllocated(), protocmp.Transform()); diff != "" { + t.Errorf("allocated mismatch after release (-want +got):\n%s", diff) + } + }) + + t.Run("ReleaseActorFromWorker_AlreadyReleased", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + worker, err := s.GetWorker(ctx, testWorkerName) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + // Release runs on paths that retry, so a second pass has to converge + // rather than fail -- and must not advance the Worker either. + released, err := s.ReleaseActorFromWorker(ctx, testWorkerName, "uid-1") + if err != nil { + t.Fatalf("ReleaseActorFromWorker failed: %v", err) + } + if released != nil { + t.Error("ReleaseActorFromWorker reported releasing an assignment that was never there") + } + after, err := s.GetWorker(ctx, testWorkerName) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + if after.GetMetadata().GetVersion() != worker.GetMetadata().GetVersion() { + t.Errorf("worker version moved from %d to %d on a release that freed nothing", + worker.GetMetadata().GetVersion(), after.GetMetadata().GetVersion()) + } + }) + + t.Run("ListWorkerAssignments_ScopedToOneWorker", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + for _, pod := range []string{"pod-1", "pod-2"} { + if _, err := s.CreateWorker(ctx, newTestWorker("worker-"+pod, pod)); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + } + bind(t, s, "worker-pod-1", newTestAssignment("uid-1", 0, 0)) + bind(t, s, "worker-pod-1", newTestAssignment("uid-2", 0, 0)) + bind(t, s, "worker-pod-2", newTestAssignment("uid-3", 0, 0)) + + assignmentsPage, err := s.ListWorkerAssignments(ctx, "worker-pod-1", store.ListOptions{}) + if err != nil { + t.Fatalf("ListWorkerAssignments failed: %v", err) + } + assignments := assignmentsPage.Items + var got []string + for _, assignment := range assignments { + got = append(got, assignment.GetActorUid()) + } + if diff := cmp.Diff([]string{"uid-1", "uid-2"}, got); diff != "" { + t.Errorf("assignments of worker-pod-1 (-want +got):\n%s", diff) + } + + // An Actor is hosted by one Worker, and only that Worker's list has it. + if _, err := s.GetWorkerAssignment(ctx, "worker-pod-1", "uid-3"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("GetWorkerAssignment found another worker's actor: %v", err) + } + }) + + // A Worker holds far more Actors than a page, so a caller that stops at the + // first page sees an arbitrary subset. Walking the token must yield each + // assignment exactly once. + t.Run("ListWorkerAssignments_Paginates", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + const total = 5 + for i := range total { + bind(t, s, testWorkerName, newTestAssignment(fmt.Sprintf("uid-%d", i), 1, 1)) + } + + seen := map[string]int{} + pages := 0 + for token := ""; ; pages++ { + page, err := s.ListWorkerAssignments(ctx, testWorkerName, store.ListOptions{PageSize: 2, PageToken: token}) + if err != nil { + t.Fatalf("ListWorkerAssignments page %d failed: %v", pages, err) + } + if got := len(page.Items); got > 2 { + t.Fatalf("page %d returned %d assignments, want at most the requested 2", pages, got) + } + for _, a := range page.Items { + seen[a.GetActorUid()]++ + } + if !page.HasNextPage() { + break + } + token = page.NextPageToken + } + if len(seen) != total { + t.Errorf("walking the pages saw %d distinct assignments, want %d", len(seen), total) + } + for uid, n := range seen { + if n != 1 { + t.Errorf("assignment %s appeared %d times across pages, want once", uid, n) + } + } + }) + + t.Run("ListWorkerAssignments_Empty", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + assignmentsPage, err := s.ListWorkerAssignments(ctx, testWorkerName, store.ListOptions{}) + if err != nil { + t.Fatalf("ListWorkerAssignments failed: %v", err) + } + assignments := assignmentsPage.Items + if len(assignments) != 0 { + t.Errorf("an idle worker reported %d assignments, want 0", len(assignments)) + } + }) + + t.Run("FindWorkerHostingActor", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + if _, err := s.FindWorkerHostingActor(ctx, "uid-1"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("FindWorkerHostingActor before any bind = %v, want ErrNotFound", err) + } + + bind(t, s, testWorkerName, newTestAssignment("uid-1", 0, 0)) + got, err := s.FindWorkerHostingActor(ctx, "uid-1") + if err != nil { + t.Fatalf("FindWorkerHostingActor failed: %v", err) + } + if got != testWorkerName { + t.Errorf("FindWorkerHostingActor = %q, want %q", got, testWorkerName) + } + + // Released, so nothing hosts it -- the recovery this exists for must + // not resurrect a placement that is over. + if _, err := s.ReleaseActorFromWorker(ctx, testWorkerName, "uid-1"); err != nil { + t.Fatalf("ReleaseActorFromWorker failed: %v", err) + } + if _, err := s.FindWorkerHostingActor(ctx, "uid-1"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("FindWorkerHostingActor after release = %v, want ErrNotFound", err) + } + }) + + t.Run("DeleteWorker_DropsItsAssignments", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + bind(t, s, testWorkerName, newTestAssignment("uid-1", 0, 0)) + if _, err := s.DeleteWorker(ctx, testWorkerName, store.DeletePreconditions{}); err != nil { + t.Fatalf("DeleteWorker failed: %v", err) + } + + // A Worker's assignments cannot outlive it: the pod is gone, so nothing + // is hosted, and a leftover row would keep an Actor looking placed. + assignmentsPage, err := s.ListWorkerAssignments(ctx, testWorkerName, store.ListOptions{}) + if err != nil { + t.Fatalf("ListWorkerAssignments failed: %v", err) + } + assignments := assignmentsPage.Items + if len(assignments) != 0 { + t.Errorf("deleted worker still holds %d assignments", len(assignments)) + } + if _, err := s.FindWorkerHostingActor(ctx, "uid-1"); !errors.Is(err, store.ErrNotFound) { + t.Errorf("FindWorkerHostingActor still names a deleted worker: %v", err) + } + }) + + // TestAllocatedTracksAssignmentsUnderChurn is the test a stored total needs, + // because the risk of storing one is not that the arithmetic is wrong once + // -- it is that some sequence of binds and releases leaves it disagreeing + // with what it summarizes, quietly, forever after. + // + // So: churn a worker through hundreds of binds, rebinds and releases, and + // after every one require the total to equal the assignments. + t.Run("Allocated_TracksAssignmentsUnderChurn", func(t *testing.T) { + s := setup(t) + ctx := context.Background() + + if _, err := s.CreateWorker(ctx, newTestWorker(testWorkerName, "pod-1")); err != nil { + t.Fatalf("CreateWorker failed: %v", err) + } + + // Deterministic, so a failure reproduces. + rng := rand.New(rand.NewPCG(1, 2)) + for step := range 300 { + actorUID := fmt.Sprintf("uid-%d", rng.IntN(12)) + + if rng.IntN(3) < 2 { + // Bind, sometimes over an actor already there and at a different + // size, which is the case that has to subtract before it adds. + assignment := newTestAssignment(actorUID, int64(rng.IntN(4)+1)*500, int64(rng.IntN(4)+1)<<24) + if err := s.BindActorToWorker(ctx, testWorkerName, assignment, nil); err != nil { + t.Fatalf("step %d: BindActorToWorker(%s) failed: %v", step, actorUID, err) + } + } else if _, err := s.ReleaseActorFromWorker(ctx, testWorkerName, actorUID); err != nil { + t.Fatalf("step %d: ReleaseActorFromWorker(%s) failed: %v", step, actorUID, err) + } + + worker, err := s.GetWorker(ctx, testWorkerName) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + assignmentsPage, err := s.ListWorkerAssignments(ctx, testWorkerName, store.ListOptions{}) + if err != nil { + t.Fatalf("ListWorkerAssignments failed: %v", err) + } + assignments := assignmentsPage.Items + want, err := resources.SumAllocated(assignments) + if err != nil { + t.Fatalf("summing assignments: %v", err) + } + if diff := cmp.Diff(want, worker.GetStatus().GetAllocation().GetAllocated(), protocmp.Transform()); diff != "" { + t.Fatalf("after step %d (actor %s) the total no longer matches the assignments (-want +got):\n%s", step, actorUID, diff) + } + } + + // And it must come back to exactly zero, not merely to something small: + // a total drifting by a little per cycle is what this guards against. + assignmentsPage, err := s.ListWorkerAssignments(ctx, testWorkerName, store.ListOptions{}) + if err != nil { + t.Fatalf("ListWorkerAssignments failed: %v", err) + } + assignments := assignmentsPage.Items + for _, assignment := range assignments { + if _, err := s.ReleaseActorFromWorker(ctx, testWorkerName, assignment.GetActorUid()); err != nil { + t.Fatalf("ReleaseActorFromWorker failed: %v", err) + } + } + worker, err := s.GetWorker(ctx, testWorkerName) + if err != nil { + t.Fatalf("GetWorker failed: %v", err) + } + if got := worker.GetStatus().GetAllocation().GetAllocated(); got.GetActors() != 0 || got.GetResources() != nil { + t.Errorf("after releasing everything the total is %v, want all zero", got) + } + }) +} + func runAtespaceContractTests(t *testing.T, setup func(t *testing.T) store.Interface) { t.Helper() @@ -2241,7 +2910,7 @@ func runUnknownFieldContractTests(t *testing.T, setup func(t *testing.T) store.I worker := withUnknownField(newTestWorker(testWorkerName, "pod-1")) withUnknownField(worker.Metadata) - withUnknownField(worker.Capacity) + withUnknownField(worker.Status.Allocation.Capacity) created, err := s.CreateWorker(ctx, worker) if err != nil { t.Fatalf("CreateWorker failed: %v", err) diff --git a/cmd/ateapi/internal/workercache/workercache_test.go b/cmd/ateapi/internal/workercache/workercache_test.go index 48b2d55812..8988aa038f 100644 --- a/cmd/ateapi/internal/workercache/workercache_test.go +++ b/cmd/ateapi/internal/workercache/workercache_test.go @@ -113,19 +113,15 @@ func TestCache_UpdatedEvent_NewerVersionApplied(t *testing.T) { } updated := makeWorker("ns", "pod1", 2) - updated.Status.Assignment = &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "actor-1"}, - ActorUid: "actor-1-uid", - } + updated.Status.Allocation.Allocated = &ateapipb.WorkerResources{Actors: 1} fs.send(store.WorkerEvent{Type: store.WorkerEventUpdated, Worker: updated}) eventually(t, func() bool { workers, err := c.Workers() - if err != nil || len(workers) != 1 || workers[0].GetStatus().GetAssignment() == nil { + if err != nil || len(workers) != 1 { return false } - wass := workers[0].GetStatus().GetAssignment() - return wass.Actor.Name == "actor-1" && wass.ActorUid == "actor-1-uid" + return workers[0].GetStatus().GetAllocation().GetAllocated().GetActors() == 1 }, 2*time.Second) got, _ := c.Workers() @@ -146,10 +142,7 @@ func TestCache_UpdatedEvent_OlderVersionIgnored(t *testing.T) { // Send a stale update followed by a sentinel we can detect. stale := makeWorker("ns", "pod1", 3) - stale.Status.Assignment = &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "stale-actor"}, - ActorUid: "stale-actor-uid", - } + stale.Status.Allocation.Allocated = &ateapipb.WorkerResources{Actors: 7} fs.send(store.WorkerEvent{Type: store.WorkerEventUpdated, Worker: stale}) sentinel := makeWorker("ns", "pod2", 1) @@ -455,7 +448,7 @@ func makeWorker(namespace, pod string, version int64) *ateapipb.Worker { WorkerNamespace: namespace, WorkerPod: pod, WorkerPodUid: workerName(namespace, pod), - Status: &ateapipb.WorkerStatus{}, + Status: &ateapipb.WorkerStatus{Allocation: &ateapipb.WorkerAllocation{}}, } } diff --git a/cmd/ateapi/internal/workerservice/capacity.go b/cmd/ateapi/internal/workerservice/capacity.go new file mode 100644 index 0000000000..eb91f322fe --- /dev/null +++ b/cmd/ateapi/internal/workerservice/capacity.go @@ -0,0 +1,136 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package workerservice serves the RPCs a Worker uses to tell the control +// plane about itself. +package workerservice + +import ( + "context" + "errors" + "fmt" + "log/slog" + "maps" + "slices" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// Server implements ateapipb.WorkerServiceServer. +type Server struct { + ateapipb.UnimplementedWorkerServiceServer + + // store is where a Worker's reported capacity is recorded, and the + // authoritative state the report is authorized against. + store store.Interface +} + +var _ ateapipb.WorkerServiceServer = (*Server)(nil) + +func New(store store.Interface) *Server { + return &Server{store: store} +} + +// SetWorkerCapacity records a Worker's reported capacity. As with MintCert, +// the caller must be an atelet running on the Worker's node. +func (s *Server) SetWorkerCapacity(ctx context.Context, req *ateapipb.SetWorkerCapacityRequest) (*ateapipb.SetWorkerCapacityResponse, error) { + caller, err := ateletauth.Authenticate(ctx) + if err != nil { + return nil, err + } + // Workers are global-scoped, so the reference carries no atespace. + if errs := resources.ValidateGlobalObjectRef(req.GetWorker(), field.NewPath("worker")); len(errs) > 0 { + return nil, status.Errorf(codes.InvalidArgument, "invalid worker: %v", errs.ToAggregate()) + } + reported := req.GetCapacity() + if reported == nil { + return nil, status.Error(codes.InvalidArgument, "capacity is required") + } + if err := validateReportedCapacity(reported); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid capacity: %v", err) + } + name := req.GetWorker().GetName() + + // Use authoritative state to authorize the write. + worker, err := s.store.GetWorker(ctx, name) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return nil, status.Errorf(codes.NotFound, "Worker %s not found", name) + } + return nil, fmt.Errorf("while fetching worker %s: %w", name, err) + } + if worker.GetNodeName() != caller.NodeName { + // Do not disclose Workers on other nodes. + slog.WarnContext(ctx, "Refusing a capacity report for a worker on another node", + slog.String("worker", name), + slog.String("worker_node", worker.GetNodeName()), + slog.String("caller_node", caller.NodeName), + slog.String("caller_pod", caller.PodName)) + return nil, status.Errorf(codes.NotFound, "Worker %s not found", name) + } + + if proto.Equal(worker.GetStatus().GetAllocation().GetCapacity(), reported) { + return &ateapipb.SetWorkerCapacityResponse{Worker: worker}, nil + } + + updated, err := s.store.UpdateWorker(ctx, name, store.PreconditionFrom(worker), func(toUpdate *ateapipb.Worker) error { + // Replaces rather than merges: a Worker reports everything it has, so a + // dimension this report leaves out is one it no longer supplies. + resources.Allocation(toUpdate).Capacity = reported + return nil + }) + switch { + case err == nil: + case errors.Is(err, store.ErrNotFound): + return nil, status.Errorf(codes.NotFound, "Worker %s not found", name) + case errors.Is(err, store.ErrUIDConflict), errors.Is(err, store.ErrVersionConflict): + return nil, status.Error(codes.Aborted, "concurrent update conflict, please retry") + default: + return nil, fmt.Errorf("while recording capacity for worker %s: %w", name, err) + } + slog.InfoContext(ctx, "Worker reported its capacity", + slog.String("worker", name), + slog.String("was", worker.GetStatus().GetAllocation().GetCapacity().String()), + slog.String("now", updated.GetStatus().GetAllocation().GetCapacity().String())) + return &ateapipb.SetWorkerCapacityResponse{Worker: updated}, nil +} + +// validateReportedCapacity rejects a report that cannot mean anything. A report +// is written straight to the store, so it does not pass the declarative +// validation an UpdateWorker request would, and an unchecked ceiling persists. +// A negative one is the costly case: placement asks whether allocated is below +// capacity, which is false for every Actor, so the Worker silently never takes +// another one. +func validateReportedCapacity(reported *ateapipb.WorkerResources) error { + if reported.GetActors() < 0 { + return fmt.Errorf("actors is %d, must not be negative", reported.GetActors()) + } + quantities, err := resources.ParseQuantities(reported.GetResources()) + if err != nil { + return err + } + for _, name := range slices.Sorted(maps.Keys(quantities)) { + if quantity := quantities[name]; quantity.Sign() < 0 { + return fmt.Errorf("%s is %s, must not be negative", name, quantity.String()) + } + } + return nil +} diff --git a/cmd/ateapi/internal/workerservice/capacity_test.go b/cmd/ateapi/internal/workerservice/capacity_test.go new file mode 100644 index 0000000000..602a040e42 --- /dev/null +++ b/cmd/ateapi/internal/workerservice/capacity_test.go @@ -0,0 +1,212 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workerservice + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/testing/protocmp" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/ateletauth/ateletauthtest" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + capWorkerName = "8f1c2d34-5e6a-4b7c-9d8e-0f1a2b3c4d5e" + capNode = "node-1" +) + +// seedReportedWorker registers a Worker on nodeName that has already reported +// capacity: identity from the pod, the way the syncer writes it, plus the +// result of an earlier report. A Worker that has never reported carries none, +// so what a fresh report replaces is what this seeds. +func seedReportedWorker(t *testing.T, st store.Interface, nodeName string, capacity *ateapipb.WorkerResources) *ateapipb.Worker { + t.Helper() + created, err := st.CreateWorker(context.Background(), &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: capWorkerName}, + WorkerNamespace: "ate-system", + WorkerPool: "pool-1", + WorkerPod: "worker-pod-1", + WorkerPodUid: capWorkerName, + NodeName: nodeName, + Ip: "10.1.2.3", + SandboxClass: "gvisor", + Status: &ateapipb.WorkerStatus{State: ateapipb.WorkerState_WORKER_STATE_ACTIVE, Allocation: &ateapipb.WorkerAllocation{Capacity: capacity}}, + }) + if err != nil { + t.Fatalf("seeding worker: %v", err) + } + return created +} + +func setRequest(actors int32) *ateapipb.SetWorkerCapacityRequest { + return &ateapipb.SetWorkerCapacityRequest{ + Worker: &ateapipb.ObjectRef{Name: capWorkerName}, + Capacity: &ateapipb.WorkerResources{Actors: actors}, + } +} + +// The point of the whole path: a Worker moves from what it reported before to +// what its ateom reports now. +func TestSetWorkerCapacity(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st) + seedReportedWorker(t, st, capNode, &ateapipb.WorkerResources{Actors: 1, Resources: resources.CPUMemory(2000, 0)}) + + got, err := s.SetWorkerCapacity(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, capNode)), setRequest(4094)) + if err != nil { + t.Fatalf("SetWorkerCapacity() failed: %v", err) + } + if want := int32(4094); got.GetWorker().GetStatus().GetAllocation().GetCapacity().GetActors() != want { + t.Errorf("capacity.actors = %d, want %d", got.GetWorker().GetStatus().GetAllocation().GetCapacity().GetActors(), want) + } + // A report replaces what is recorded. The Worker reports everything it has, + // so a dimension this one leaves out is one it no longer supplies -- keeping + // the old value would advertise compute nothing claims to have. + if got := got.GetWorker().GetStatus().GetAllocation().GetCapacity().GetResources(); got != nil { + t.Errorf("capacity resources = %v, want the report's own (none)", got) + } +} + +// An atelet speaks for the Workers it herds and no others. A Worker on another +// node is reported as absent rather than forbidden, so a caller learns nothing +// about what runs elsewhere. +func TestSetWorkerCapacity_OtherNodeIsNotFound(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st) + seedReportedWorker(t, st, capNode, &ateapipb.WorkerResources{Actors: 1}) + + _, err := s.SetWorkerCapacity(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, "some-other-node")), setRequest(4094)) + if got := status.Code(err); got != codes.NotFound { + t.Fatalf("code = %v (err %v), want NotFound", got, err) + } + + // And the report must not have landed. + after, err := st.GetWorker(context.Background(), capWorkerName) + if err != nil { + t.Fatalf("GetWorker: %v", err) + } + if got := after.GetStatus().GetAllocation().GetCapacity().GetActors(); got != 1 { + t.Errorf("capacity.actors = %d, want 1 unchanged", got) + } +} + +// Re-sending the same capacity is not an update. An ateom reports once, but it +// retries until accepted and reports again if it restarts, so a repeat must not +// churn the Worker's version. +func TestSetWorkerCapacity_UnchangedDoesNotWrite(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st) + seeded := seedReportedWorker(t, st, capNode, &ateapipb.WorkerResources{Actors: 4094}) + + for range 3 { + if _, err := s.SetWorkerCapacity(ateletauthtest.ContextWith(ateletauthtest.CertOn(t, capNode)), setRequest(4094)); err != nil { + t.Fatalf("SetWorkerCapacity() failed: %v", err) + } + } + after, err := st.GetWorker(context.Background(), capWorkerName) + if err != nil { + t.Fatalf("GetWorker: %v", err) + } + if got, want := after.GetMetadata().GetVersion(), seeded.GetMetadata().GetVersion(); got != want { + t.Errorf("version = %d after three identical reports, want %d unchanged", got, want) + } +} + +func TestSetWorkerCapacity_Errors(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st) + seedReportedWorker(t, st, capNode, &ateapipb.WorkerResources{Actors: 1}) + authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, capNode)) + + tests := []struct { + name string + ctx context.Context + req *ateapipb.SetWorkerCapacityRequest + want codes.Code + }{ + {"unauthenticated", ateletauthtest.ContextWith(nil), setRequest(2), codes.Unauthenticated}, + {"no worker ref", authed, &ateapipb.SetWorkerCapacityRequest{ + Capacity: &ateapipb.WorkerResources{Actors: 2}, + }, codes.InvalidArgument}, + {"no capacity", authed, &ateapipb.SetWorkerCapacityRequest{ + Worker: &ateapipb.ObjectRef{Name: capWorkerName}, + }, codes.InvalidArgument}, + {"absent worker", authed, &ateapipb.SetWorkerCapacityRequest{ + Worker: &ateapipb.ObjectRef{Name: "3b9f1e77-2c4d-4a80-91be-6d5c8f0a7e21"}, + Capacity: &ateapipb.WorkerResources{Actors: 2}, + }, codes.NotFound}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := s.SetWorkerCapacity(tc.ctx, tc.req) + if got := status.Code(err); got != tc.want { + t.Errorf("code = %v (err %v), want %v", got, err, tc.want) + } + }) + } +} + +// A report goes straight to the store, so nothing else checks it. A negative +// ceiling is the case that matters: placement asks whether allocated is below +// capacity, so the Worker would take no Actor ever again. +func TestSetWorkerCapacity_RejectsNonsense(t *testing.T) { + st, cleanup := storetest.SetupTestStore(t) + defer cleanup() + s := New(st) + seeded := seedReportedWorker(t, st, capNode, &ateapipb.WorkerResources{Actors: 4094}) + authed := ateletauthtest.ContextWith(ateletauthtest.CertOn(t, capNode)) + + for _, tc := range []struct { + name string + capacity *ateapipb.WorkerResources + }{ + {"negative ceiling", &ateapipb.WorkerResources{Actors: -1}}, + {"int32 underflow", &ateapipb.WorkerResources{Actors: -2147483648}}, + {"negative quantity", &ateapipb.WorkerResources{Resources: resources.CPUMemory(-1, 0)}}, + {"unparseable quantity", &ateapipb.WorkerResources{ + Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{{Name: "cpu", Quantity: "lots"}}}, + }}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := s.SetWorkerCapacity(authed, &ateapipb.SetWorkerCapacityRequest{ + Worker: &ateapipb.ObjectRef{Name: capWorkerName}, + Capacity: tc.capacity, + }) + if got := status.Code(err); got != codes.InvalidArgument { + t.Fatalf("code = %v (err %v), want %v", got, err, codes.InvalidArgument) + } + }) + } + + after, err := st.GetWorker(context.Background(), capWorkerName) + if err != nil { + t.Fatalf("GetWorker: %v", err) + } + if diff := cmp.Diff(seeded.GetStatus().GetAllocation().GetCapacity(), after.GetStatus().GetAllocation().GetCapacity(), protocmp.Transform()); diff != "" { + t.Errorf("capacity changed despite every report being refused (-want +got):\n%s", diff) + } +} diff --git a/cmd/ateapi/internal/workerservice/main_test.go b/cmd/ateapi/internal/workerservice/main_test.go new file mode 100644 index 0000000000..04d3bc84b9 --- /dev/null +++ b/cmd/ateapi/internal/workerservice/main_test.go @@ -0,0 +1,25 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package workerservice + +import ( + "testing" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" +) + +func TestMain(m *testing.M) { + storetest.RunTests(m) +} diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index 658e1b075d..a5ca440222 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -33,6 +33,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/atepg" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/workerservice" "github.com/agent-substrate/substrate/internal/ateapiauth" "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/credbundle" @@ -239,6 +240,7 @@ func main() { reflection.Register(mux) ateapipb.RegisterControlServer(mux, controlSrv) ateapipb.RegisterActorIdentityServer(mux, actorIdentitySrv) + ateapipb.RegisterWorkerServiceServer(mux, workerservice.New(persistence)) readiness := &serverboot.Readiness{} go serverboot.StartMetricsServer(ctx, serverboot.MetricsServerOptions{ diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index c2fdf57773..1524bd1fad 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -24,6 +24,7 @@ import ( corev1ac "k8s.io/client-go/applyconfigurations/core/v1" metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" + "github.com/agent-substrate/substrate/internal/ateomcapacity" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/deviceplugin" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -68,6 +69,7 @@ const ( atunnelIdentityMountPath = "/run/podidentity.podcert.ate.dev" atunnelEgressTrustVolume = "atunnel-egress-trust" atunnelEgressTrustMountPath = "/run/servicedns.podcert.ate.dev" + ateomCapacityVolume = "ateom-capacity" ) // buildDeploymentApplyConfig constructs the SSA apply configuration for the @@ -118,6 +120,10 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin WithSecurityContext(ateomSecurityContext(wp.Spec.SandboxClass)). WithEnv(ateomContainerEnv(otel)...). WithVolumeMounts( + corev1ac.VolumeMount(). + WithName(ateomCapacityVolume). + WithMountPath(ateomcapacity.CapacityMountPath). + WithReadOnly(true), corev1ac.VolumeMount(). WithName("run-ateom"). WithMountPath(ateompath.BasePath). @@ -137,6 +143,13 @@ func buildDeploymentApplyConfig(wp *atev1alpha1.WorkerPool, otel ateomOTelSettin WithRunAsUser(0). WithRunAsGroup(0)). WithVolumes( + corev1ac.Volume(). + WithName(ateomCapacityVolume). + WithDownwardAPI(corev1ac.DownwardAPIVolumeSource(). + WithItems( + resourceFieldRefFile(ateomcapacity.CPULimitFile, "limits.cpu", milliCores), + resourceFieldRefFile(ateomcapacity.MemoryLimitFile, "limits.memory", wholeBytes), + )), corev1ac.Volume(). WithName("run-ateom"). WithHostPath(corev1ac.HostPathVolumeSource(). @@ -238,6 +251,24 @@ func ateomContainerEnv(otel ateomOTelSettings) []*corev1ac.EnvVarApplyConfigurat return envs } +// Divisors for resourceFieldRefFile. The downward API reports +// ceil(limit/divisor), so these are the units the value arrives in; the default +// divisor of one core would round a fractional CPU limit up to a whole one. +const ( + milliCores = "1m" + wholeBytes = "1" +) + +// resourceFieldRefFile projects a container resource in units of divisor. +func resourceFieldRefFile(path, resourceName, divisor string) *corev1ac.DownwardAPIVolumeFileApplyConfiguration { + return corev1ac.DownwardAPIVolumeFile(). + WithPath(path). + WithResourceFieldRef(corev1ac.ResourceFieldSelector(). + WithContainerName("ateom"). + WithResource(resourceName). + WithDivisor(resource.MustParse(divisor))) +} + func fieldRefEnv(name, fieldPath string) *corev1ac.EnvVarApplyConfiguration { return corev1ac.EnvVar(). WithName(name). diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 85bff190f4..7c7b4fe316 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -28,6 +28,7 @@ import ( corev1ac "k8s.io/client-go/applyconfigurations/core/v1" metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" + "github.com/agent-substrate/substrate/internal/ateomcapacity" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/deviceplugin" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -676,6 +677,13 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf WithRunAsUser(0). WithRunAsGroup(0)). WithVolumes( + corev1ac.Volume(). + WithName(ateomCapacityVolume). + WithDownwardAPI(corev1ac.DownwardAPIVolumeSource(). + WithItems( + resourceFieldRefFile(ateomcapacity.CPULimitFile, "limits.cpu", milliCores), + resourceFieldRefFile(ateomcapacity.MemoryLimitFile, "limits.memory", wholeBytes), + )), corev1ac.Volume(). WithName("run-ateom"). WithHostPath(corev1ac.HostPathVolumeSource(). @@ -758,6 +766,10 @@ func expectedDeploymentApplyConfig(mutatePodSpec func(*corev1ac.PodSpecApplyConf WithFieldPath("metadata.uid"))), ). WithVolumeMounts( + corev1ac.VolumeMount(). + WithName(ateomCapacityVolume). + WithMountPath(ateomcapacity.CapacityMountPath). + WithReadOnly(true), corev1ac.VolumeMount(). WithName("run-ateom"). WithMountPath(ateompath.BasePath). diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go index c1f3b8b6f7..3c05c01c37 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller_test.go @@ -137,10 +137,11 @@ func TestWorkerPoolCreatesDeployment(t *testing.T) { if len(dep.OwnerReferences) == 0 || dep.OwnerReferences[0].Name != wp.Name { return false, nil } - return len(dep.Spec.Template.Spec.Volumes) == 3 && - dep.Spec.Template.Spec.Volumes[0].Name == "run-ateom" && - dep.Spec.Template.Spec.Volumes[1].Name == atunnelIdentityVolume && - dep.Spec.Template.Spec.Volumes[2].Name == atunnelEgressTrustVolume, nil + return len(dep.Spec.Template.Spec.Volumes) == 4 && + dep.Spec.Template.Spec.Volumes[0].Name == ateomCapacityVolume && + dep.Spec.Template.Spec.Volumes[1].Name == "run-ateom" && + dep.Spec.Template.Spec.Volumes[2].Name == atunnelIdentityVolume && + dep.Spec.Template.Spec.Volumes[3].Name == atunnelEgressTrustVolume, nil }) } diff --git a/cmd/atecontroller/internal/workersync/capacity_test.go b/cmd/atecontroller/internal/workersync/capacity_test.go deleted file mode 100644 index d3503910f3..0000000000 --- a/cmd/atecontroller/internal/workersync/capacity_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package workersync - -import ( - "testing" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -// TestWorkerCapacity covers the worker-side extraction: capacity comes from the -// ateom container's limits, not the pod total, and other containers are ignored. -func TestWorkerCapacity(t *testing.T) { - pod := func(ctrs ...corev1.Container) *corev1.Pod { - return &corev1.Pod{Spec: corev1.PodSpec{Containers: ctrs}} - } - limited := func(name, cpu, mem string) corev1.Container { - lim := corev1.ResourceList{} - if cpu != "" { - lim[corev1.ResourceCPU] = resource.MustParse(cpu) - } - if mem != "" { - lim[corev1.ResourceMemory] = resource.MustParse(mem) - } - return corev1.Container{Name: name, Resources: corev1.ResourceRequirements{Limits: lim}} - } - - tests := []struct { - name string - pod *corev1.Pod - wantCPU int64 - wantMemory int64 - }{ - { - name: "no ateom container yields zero", - pod: pod(limited("sidecar", "1", "1Gi")), - wantCPU: 0, - wantMemory: 0, - }, - { - name: "ateom container limits become capacity", - pod: pod(limited(ateomContainerName, "4", "8Gi")), - wantCPU: 4000, - wantMemory: 8 << 30, - }, - { - name: "only the ateom container counts, not the pod total", - pod: pod(limited("sidecar", "16", "64Gi"), limited(ateomContainerName, "2", "2Gi")), - wantCPU: 2000, - wantMemory: 2 << 30, - }, - { - name: "unset dimension reports zero", - pod: pod(limited(ateomContainerName, "2", "")), - wantCPU: 2000, - wantMemory: 0, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got := workerCapacity(tc.pod) - if got.GetCpuMilli() != tc.wantCPU || got.GetMemoryBytes() != tc.wantMemory { - t.Fatalf("workerCapacity() = (%d, %d), want (%d, %d)", - got.GetCpuMilli(), got.GetMemoryBytes(), tc.wantCPU, tc.wantMemory) - } - }) - } -} diff --git a/cmd/atecontroller/internal/workersync/syncer.go b/cmd/atecontroller/internal/workersync/syncer.go index 4c020952c2..97fbd77d5e 100644 --- a/cmd/atecontroller/internal/workersync/syncer.go +++ b/cmd/atecontroller/internal/workersync/syncer.go @@ -265,7 +265,10 @@ func (s *WorkerPoolSyncer) createOrUpdateWorker(ctx context.Context, key workerK NodeName: pod.Spec.NodeName, SandboxClass: string(pool.Spec.SandboxClass), Labels: pool.GetLabels(), - Capacity: workerCapacity(pod), + // Capacity is the Worker's to report, not the syncer's to infer + // from the pod: it is what the ateom can actually supply. Until + // that report lands, CreateWorker's reified ceiling holds the + // Worker to a single Actor. } // status is output-only: CreateWorker sets STATE_ACTIVE itself. // @@ -327,38 +330,6 @@ func isWorkerEligible(pod *corev1.Pod) bool { return false } -// ateomContainerName is the name of the container in a worker pod that hosts the -// actor's sandbox; its resource limits bound what an actor placed here can use. -const ateomContainerName = "ateom" - -// workerCapacity returns the worker pod's capacity for hosting an actor — CPU -// in millicores and memory in bytes — taken from the ateom container's resource -// limits. A dimension the pod does not limit reports 0, which the scheduler -// treats as "unknown" (unconstrained); a pod that limits neither reports nil -// rather than an all-zero message that says the same thing. The actor sandbox -// runs nested in the ateom container's cgroup, so that container's limits — not -// the pod total — are the relevant envelope. -func workerCapacity(pod *corev1.Pod) *ateapipb.WorkerCapacity { - var capacity ateapipb.WorkerCapacity - for i := range pod.Spec.Containers { - c := &pod.Spec.Containers[i] - if c.Name != ateomContainerName { - continue - } - if v := c.Resources.Limits.Cpu(); v != nil { - capacity.CpuMilli = v.MilliValue() - } - if v := c.Resources.Limits.Memory(); v != nil { - capacity.MemoryBytes = v.Value() - } - break - } - if capacity.CpuMilli == 0 && capacity.MemoryBytes == 0 { - return nil - } - return &capacity -} - // markWorkerDraining transitions a worker to STATE_DRAINING so the scheduler // stops routing new actors to it while its pod is Terminating. DrainWorker is // idempotent, so a worker already draining costs nothing. If the worker is diff --git a/cmd/atecontroller/internal/workersync/syncer_test.go b/cmd/atecontroller/internal/workersync/syncer_test.go index fd39e17982..78dda1c0b7 100644 --- a/cmd/atecontroller/internal/workersync/syncer_test.go +++ b/cmd/atecontroller/internal/workersync/syncer_test.go @@ -30,6 +30,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -255,6 +256,41 @@ func TestSyncer_OmittedFields(t *testing.T) { } } +// Capacity belongs to the Worker, which sets it over WorkerService. The +// syncer must not infer one from the pod, even when the ateom container spells +// its limits out: the pod's limits bound the sandbox, but what the ateom can +// actually supply to Actors is the ateom's to say. +func TestSyncer_DoesNotInferCapacityFromThePod(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + ns, podName, poolName := "ns-syncer-capacity", "worker-unit-1", "pool1" + + api := newFakeControl() + fakeK8s := setupSyncerTest(t, ctx, api, workerPool(ns, poolName, "gvisor", nil)) + + pod := workerPod(ns, podName, poolName, testPodUID, "127.0.0.1") + pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{ + Name: "ateom", + Image: "ateom", + Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + corev1.ResourceMemory: resource.MustParse("8Gi"), + }}, + }) + if _, err := fakeK8s.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil { + t.Fatalf("create pod: %v", err) + } + + // The registry here is a fake, so what it holds is exactly what the syncer + // sent. The syncer writes no capacity: it comes from the Worker's own + // report, which is the ateom's to make. + got := waitForWorker(t, ctx, api, testPodUID, func(w *ateapipb.Worker) bool { return w != nil }) + if got.GetStatus().GetAllocation().GetCapacity() != nil { + t.Errorf("worker capacity = %v, want none", got.GetStatus().GetAllocation().GetCapacity()) + } +} + // TestSyncer_SoftDelete_MarksDraining verifies that a pod entering Terminating // (DeletionTimestamp set) flips its worker to STATE_DRAINING without deleting // the worker record — the actor inside is still gracefully shutting down. diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 7a1f06335c..0314faa82e 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -364,6 +364,9 @@ func main() { ateletpb.RegisterCredentialBrokerServer(brokerServer, &credentialBroker{ actorIdentityClient: ateapipb.NewActorIdentityClient(ateapiConn), }) + ateletpb.RegisterWorkerCapacityServer(brokerServer, &workerCapacityService{ + workers: ateapipb.NewWorkerServiceClient(ateapiConn), + }) go func() { if err := brokerServer.Serve(brokerLis); err != nil { serverboot.Fatal(ctx, "Failed to serve credential broker", err) diff --git a/cmd/atelet/workercapacity.go b/cmd/atelet/workercapacity.go new file mode 100644 index 0000000000..b53a102133 --- /dev/null +++ b/cmd/atelet/workercapacity.go @@ -0,0 +1,59 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "log/slog" + + "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// workerCapacityService forwards a worker's own account of what it can supply +// to the control plane. The worker is the only thing that knows: the control +// plane sees a Pod, not what the runtime will actually give an actor. +type workerCapacityService struct { + ateletpb.UnimplementedWorkerCapacityServer + + workers ateapipb.WorkerServiceClient +} + +// SetWorkerCapacity records what the calling worker says it has. +// +// It returns the control plane's error unwrapped so the caller retries: a +// worker reports once, so an accepted call is the only thing that puts +// capacity on the Worker, and a Worker record the syncer has not created yet +// is the ordinary reason for a first attempt to fail. +func (s *workerCapacityService) SetWorkerCapacity(ctx context.Context, req *ateletpb.SetWorkerCapacityRequest) (*ateletpb.SetWorkerCapacityResponse, error) { + // Identity comes only from the mTLS certificate, never from the request: + // a worker can report its own capacity and no one else's. + workerIdentity, err := authenticatedWorkerIdentity(ctx) + if err != nil { + return nil, err + } + // Forwarded as reported: the worker speaks the vocabulary the control plane + // records, so there is nothing to translate. + if _, err := s.workers.SetWorkerCapacity(ctx, &ateapipb.SetWorkerCapacityRequest{ + // Workers are global-scoped and named by their pod UID. + Worker: &ateapipb.ObjectRef{Name: workerIdentity.PodUID}, + Capacity: req.GetCapacity(), + }); err != nil { + return nil, err + } + slog.InfoContext(ctx, "Recorded worker capacity", + slog.String("pod_uid", workerIdentity.PodUID), slog.Any("capacity", req.GetCapacity())) + return &ateletpb.SetWorkerCapacityResponse{}, nil +} diff --git a/cmd/atelet/workercapacity_test.go b/cmd/atelet/workercapacity_test.go new file mode 100644 index 0000000000..af34b2f5dd --- /dev/null +++ b/cmd/atelet/workercapacity_test.go @@ -0,0 +1,114 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/testing/protocmp" + + "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +type fakeWorkerService struct { + ateapipb.WorkerServiceClient + + got []*ateapipb.SetWorkerCapacityRequest + err error +} + +func (s *fakeWorkerService) SetWorkerCapacity(_ context.Context, in *ateapipb.SetWorkerCapacityRequest, _ ...grpc.CallOption) (*ateapipb.SetWorkerCapacityResponse, error) { + if s.err != nil { + return nil, s.err + } + s.got = append(s.got, in) + return &ateapipb.SetWorkerCapacityResponse{}, nil +} + +func TestSetWorkerCapacityRecordsWhatTheWorkerSays(t *testing.T) { + workers := &fakeWorkerService{} + svc := &workerCapacityService{workers: workers} + + ctx := workerContext(t, "pod-a") + reported := &ateapipb.WorkerResources{ + Actors: 4, + Resources: resources.CPUMemory(2000, 4294967296), + } + if _, err := svc.SetWorkerCapacity(ctx, &ateletpb.SetWorkerCapacityRequest{ + Capacity: reported, + }); err != nil { + t.Fatalf("SetWorkerCapacity() failed: %v", err) + } + + want := []*ateapipb.SetWorkerCapacityRequest{{ + // The Worker is named after the worker pod UID, taken from the + // certificate rather than the request. + Worker: &ateapipb.ObjectRef{Name: "pod-a"}, + Capacity: reported, + }} + if diff := cmp.Diff(want, workers.got, protocmp.Transform()); diff != "" { + t.Errorf("recorded capacity mismatch (-want +got):\n%s", diff) + } +} + +func TestSetWorkerCapacityOmitsUndeterminedCompute(t *testing.T) { + workers := &fakeWorkerService{} + svc := &workerCapacityService{workers: workers} + + ctx := workerContext(t, "pod-a") + if _, err := svc.SetWorkerCapacity(ctx, &ateletpb.SetWorkerCapacityRequest{Capacity: &ateapipb.WorkerResources{Actors: 1}}); err != nil { + t.Fatalf("SetWorkerCapacity() failed: %v", err) + } + + if got := workers.got[0].GetCapacity().GetResources(); got != nil { + t.Errorf("compute the worker could not determine was recorded as %v, want none", got) + } +} + +func TestSetWorkerCapacityRequiresACertificate(t *testing.T) { + workers := &fakeWorkerService{} + svc := &workerCapacityService{workers: workers} + + // No peer identity: a worker may report only what its certificate proves + // it is, so there is nothing to attribute this to. + _, err := svc.SetWorkerCapacity(context.Background(), &ateletpb.SetWorkerCapacityRequest{Capacity: &ateapipb.WorkerResources{Actors: 1}}) + if status.Code(err) != codes.Unauthenticated { + t.Errorf("unauthenticated report returned %v, want Unauthenticated", err) + } + if len(workers.got) != 0 { + t.Errorf("unauthenticated report still recorded %v", workers.got) + } +} + +func TestSetWorkerCapacitySurfacesRejection(t *testing.T) { + // The Worker record may not exist yet. The error must reach the worker so + // it retries: it reports once, so a swallowed failure leaves the Worker + // with no capacity forever. + workers := &fakeWorkerService{err: errors.New("no such worker")} + svc := &workerCapacityService{workers: workers} + + ctx := workerContext(t, "pod-a") + if _, err := svc.SetWorkerCapacity(ctx, &ateletpb.SetWorkerCapacityRequest{Capacity: &ateapipb.WorkerResources{Actors: 1}}); err == nil { + t.Error("a rejected report returned success, so the worker would not retry") + } +} diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 90cbfaec9c..ff7835cc27 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -37,6 +37,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateom-gvisor/internal/cgroupstats" "github.com/agent-substrate/substrate/internal/actorlog" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/ateomcapacity" "github.com/agent-substrate/substrate/internal/ateomnet" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/ateomstats" @@ -236,6 +237,22 @@ func do(ctx context.Context) error { svr.GracefulStop() }() + // Report what this worker can supply. Nothing else tells the control plane, + // which places no Actor here until it lands, so a worker that cannot report + // is one that will sit idle forever. Report retries every failure it can + // outlast, including the window before the Worker record exists; anything + // that reaches here is a misconfiguration no restart-in-place will fix. + go func() { + err := ateomcapacity.Report(ctx, ateomcapacity.ReportConfig{ + SocketPath: ateompath.CredentialBrokerSocket, + CredentialBundlePath: *workerCredentialBundle, + TrustBundlePath: *podIdentityTrustBundle, + }) + if err != nil && ctx.Err() == nil { + serverboot.Fatal(ctx, "Failed to report worker capacity", err) + } + }() + go serverboot.StartReadinessServer(ctx, *readinessListenAddress, readiness) if err := svr.Serve(lis); err != nil { diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 751881e37f..36978e0b9c 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -42,6 +42,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/reaper" "github.com/agent-substrate/substrate/internal/actorlog" "github.com/agent-substrate/substrate/internal/ateinterceptors" + "github.com/agent-substrate/substrate/internal/ateomcapacity" "github.com/agent-substrate/substrate/internal/ateomnet" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/atunnel" @@ -286,6 +287,22 @@ func do(ctx context.Context) error { svr.GracefulStop() }() + // Report what this worker can supply. Nothing else tells the control plane, + // which places no Actor here until it lands, so a worker that cannot report + // is one that will sit idle forever. Report retries every failure it can + // outlast, including the window before the Worker record exists; anything + // that reaches here is a misconfiguration no restart-in-place will fix. + go func() { + err := ateomcapacity.Report(ctx, ateomcapacity.ReportConfig{ + SocketPath: ateompath.CredentialBrokerSocket, + CredentialBundlePath: *workerCredentialBundle, + TrustBundlePath: *podIdentityTrustBundle, + }) + if err != nil && ctx.Err() == nil { + serverboot.Fatal(ctx, "Failed to report worker capacity", err) + } + }() + go serverboot.StartReadinessServer(ctx, *readinessListenAddress, readiness) slog.InfoContext(ctx, "ateom-microvm serving", slog.String("socket", sockPath)) diff --git a/cmd/kubectl-ate/internal/cmd/get_workers.go b/cmd/kubectl-ate/internal/cmd/get_workers.go index 260e85bbaa..582edf54f5 100644 --- a/cmd/kubectl-ate/internal/cmd/get_workers.go +++ b/cmd/kubectl-ate/internal/cmd/get_workers.go @@ -51,6 +51,7 @@ func init() { // GetWorkersRunner executes the get workers command logic. type GetWorkersRunner struct { workerLister WorkerLister + actorLister ActorLister namespace string atespace string selector string @@ -64,7 +65,7 @@ func (r *GetWorkersRunner) Run(ctx context.Context) error { if err != nil { return err } - filtered, err := filterWorkers(workers, r.namespace, r.atespace, r.selector, r.sandboxClass) + filtered, err := filterWorkers(ctx, r.actorLister, workers, r.namespace, r.atespace, r.selector, r.sandboxClass) if err != nil { return err } @@ -86,6 +87,7 @@ func runGetWorkers(cmd *cobra.Command, args []string) error { runner := &GetWorkersRunner{ workerLister: apiClient, + actorLister: apiClient, namespace: getWorkerNamespaceFlag, atespace: getWorkerAtespaceFlag, selector: getWorkerSelectorFlag, diff --git a/cmd/kubectl-ate/internal/cmd/get_workers_test.go b/cmd/kubectl-ate/internal/cmd/get_workers_test.go index d32ef975a6..90dcd8f152 100644 --- a/cmd/kubectl-ate/internal/cmd/get_workers_test.go +++ b/cmd/kubectl-ate/internal/cmd/get_workers_test.go @@ -26,19 +26,16 @@ import ( func TestGetWorkersRunner_Filters(t *testing.T) { workers := []*ateapipb.Worker{ { + Metadata: &ateapipb.ResourceMetadata{Name: "worker-1"}, WorkerNamespace: "ns-1", WorkerPool: "counter", WorkerPod: "pod-1", SandboxClass: "microvm", Labels: map[string]string{"ate.dev/worker-pool": "counter"}, - Status: &ateapipb.WorkerStatus{ - Assignment: &ateapipb.ActorAssignment{ - ActorTemplateRef: &ateapipb.ObjectRef{Atespace: "ns-1", Name: "counter"}, - Actor: &ateapipb.ObjectRef{Atespace: "space-a", Name: "actor-a"}, - }, - }, + Status: &ateapipb.WorkerStatus{Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}, Allocated: &ateapipb.WorkerResources{Actors: 1}}}, }, { + Metadata: &ateapipb.ResourceMetadata{Name: "worker-2"}, WorkerNamespace: "ns-1", WorkerPool: "other", WorkerPod: "pod-2", @@ -46,24 +43,24 @@ func TestGetWorkersRunner_Filters(t *testing.T) { Labels: map[string]string{"ate.dev/worker-pool": "other"}, }, { + Metadata: &ateapipb.ResourceMetadata{Name: "worker-3"}, WorkerNamespace: "ns-2", WorkerPool: "counter", WorkerPod: "pod-3", SandboxClass: "gvisor", Labels: map[string]string{"ate.dev/worker-pool": "counter"}, - Status: &ateapipb.WorkerStatus{ - Assignment: &ateapipb.ActorAssignment{ - ActorTemplateRef: &ateapipb.ObjectRef{Atespace: "ns-2", Name: "counter"}, - Actor: &ateapipb.ObjectRef{Atespace: "space-b", Name: "actor-b"}, - }, - }, + Status: &ateapipb.WorkerStatus{Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}, Allocated: &ateapipb.WorkerResources{Actors: 1}}}, }, } + actors := &mockActorLister{byAtespace: map[string][]*ateapipb.Actor{ + "space-a": {actorOn("space-a", "actor-a", "worker-1")}, + "space-b": {actorOn("space-b", "actor-b", "worker-3")}, + }} - header := "NAMESPACE POOL CLASS POD STATUS ASSIGNED ACTOR\n" - row1 := "ns-1 counter microvm pod-1 ASSIGNED ns-1/counter/space-a/actor-a\n" - row2 := "ns-1 other gvisor pod-2 FREE \n" - row3 := "ns-2 counter gvisor pod-3 ASSIGNED ns-2/counter/space-b/actor-b\n" + header := "NAMESPACE POOL CLASS POD STATUS\n" + row1 := "ns-1 counter microvm pod-1 ASSIGNED(1/1)\n" + row2 := "ns-1 other gvisor pod-2 FREE\n" + row3 := "ns-2 counter gvisor pod-3 ASSIGNED(1/1)\n" tests := []struct { name string @@ -77,13 +74,13 @@ func TestGetWorkersRunner_Filters(t *testing.T) { {name: "namespace", namespace: "ns-1", expected: header + row1 + row2}, {name: "atespace", atespace: "space-a", expected: header + row1}, // With no matching rows the tabwriter sizes columns to the header alone. - {name: "atespace excludes free workers", atespace: "no-such-space", expected: "NAMESPACE POOL CLASS POD STATUS ASSIGNED ACTOR\n"}, + {name: "atespace excludes free workers", atespace: "no-such-space", expected: "NAMESPACE POOL CLASS POD STATUS\n"}, {name: "selector", selector: "ate.dev/worker-pool=counter", expected: header + row1 + row3}, {name: "sandbox class", sandboxClass: "microvm", expected: header + row1}, // gvisor-only rows shrink the CLASS column to the widest survivor. - {name: "sandbox class gvisor", sandboxClass: "gvisor", expected: "NAMESPACE POOL CLASS POD STATUS ASSIGNED ACTOR\n" + - "ns-1 other gvisor pod-2 FREE \n" + - "ns-2 counter gvisor pod-3 ASSIGNED ns-2/counter/space-b/actor-b\n"}, + {name: "sandbox class gvisor", sandboxClass: "gvisor", expected: "NAMESPACE POOL CLASS POD STATUS\n" + + "ns-1 other gvisor pod-2 FREE\n" + + "ns-2 counter gvisor pod-3 ASSIGNED(1/1)\n"}, {name: "combined", namespace: "ns-1", selector: "ate.dev/worker-pool=counter", expected: header + row1}, } @@ -92,6 +89,7 @@ func TestGetWorkersRunner_Filters(t *testing.T) { var buf bytes.Buffer runner := &GetWorkersRunner{ workerLister: &mockWorkerLister{workers: workers}, + actorLister: actors, namespace: test.namespace, atespace: test.atespace, selector: test.selector, diff --git a/cmd/kubectl-ate/internal/cmd/top_workers.go b/cmd/kubectl-ate/internal/cmd/top_workers.go index 865697d885..a3b7b34f67 100644 --- a/cmd/kubectl-ate/internal/cmd/top_workers.go +++ b/cmd/kubectl-ate/internal/cmd/top_workers.go @@ -73,6 +73,7 @@ func (l *k8sPodMetricsLister) ListPodMetrics(ctx context.Context, namespace stri // TopWorkersRunner executes the top workers resource utilization command logic. type TopWorkersRunner struct { workerLister WorkerLister + actorLister ActorLister podMetricsLister PodMetricsLister namespace string atespace string @@ -87,7 +88,7 @@ func (r *TopWorkersRunner) Run(ctx context.Context) error { if err != nil { return err } - filtered, err := filterWorkers(allWorkers, r.namespace, r.atespace, r.selector, r.sandboxClass) + filtered, err := filterWorkers(ctx, r.actorLister, allWorkers, r.namespace, r.atespace, r.selector, r.sandboxClass) if err != nil { return err } @@ -114,25 +115,6 @@ func (r *TopWorkersRunner) Run(ctx context.Context) error { podName := w.GetWorkerPod() pool := w.GetWorkerPool() - status := "FREE" - assignedActor := "" - if wass := w.GetStatus().GetAssignment(); wass != nil && wass.GetActor() != nil { - status = "ASSIGNED" - if ref := wass.GetActorTemplateRef(); ref != nil { - assignedActor = fmt.Sprintf("%s/%s/%s/%s", - ref.GetAtespace(), - ref.GetName(), - wass.GetActor().GetAtespace(), - wass.GetActor().GetName(), - ) - } else { - assignedActor = fmt.Sprintf("%s/%s", - wass.GetActor().GetAtespace(), - wass.GetActor().GetName(), - ) - } - } - cpuStr := "metrics unavailable" memStr := "metrics unavailable" @@ -144,14 +126,13 @@ func (r *TopWorkersRunner) Run(ctx context.Context) error { } items = append(items, &printer.WorkerTopItem{ - Pod: podName, - Pool: pool, - Class: w.GetSandboxClass(), - Status: status, - AssignedActor: assignedActor, - CPU: cpuStr, - Memory: memStr, - Namespace: ns, + Pod: podName, + Pool: pool, + Class: w.GetSandboxClass(), + Status: printer.WorkerOccupancy(w), + CPU: cpuStr, + Memory: memStr, + Namespace: ns, }) } @@ -209,6 +190,7 @@ func runTopWorkers(cmd *cobra.Command, args []string) error { runner := &TopWorkersRunner{ workerLister: apiClient, + actorLister: apiClient, podMetricsLister: metricsLister, namespace: topWorkerNamespaceFlag, atespace: topWorkerAtespaceFlag, diff --git a/cmd/kubectl-ate/internal/cmd/top_workers_test.go b/cmd/kubectl-ate/internal/cmd/top_workers_test.go index 9ba5f5b00f..020abd5ffb 100644 --- a/cmd/kubectl-ate/internal/cmd/top_workers_test.go +++ b/cmd/kubectl-ate/internal/cmd/top_workers_test.go @@ -41,6 +41,30 @@ func (m *mockWorkerLister) ListWorkers(ctx context.Context, req *ateapipb.ListWo return &ateapipb.ListWorkersResponse{Workers: m.workers}, nil } +// mockActorLister serves the actors of one atespace at a time, as the real +// listing does, so an atespace filter sees only that atespace's placements. +type mockActorLister struct { + byAtespace map[string][]*ateapipb.Actor + err error +} + +func (m *mockActorLister) ListActors(ctx context.Context, req *ateapipb.ListActorsRequest, opts ...grpc.CallOption) (*ateapipb.ListActorsResponse, error) { + if m.err != nil { + return nil, m.err + } + return &ateapipb.ListActorsResponse{Actors: m.byAtespace[req.GetAtespace()]}, nil +} + +// actorOn is an actor of atespace placed on the named worker. +func actorOn(atespace, name, workerName string) *ateapipb.Actor { + return &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, + Status: &ateapipb.ActorStatus{ + WorkerAssignment: &ateapipb.WorkerAssignment{Worker: &ateapipb.ObjectRef{Name: workerName}}, + }, + } +} + type mockPodMetricsLister struct { metrics []metricsv1beta1.PodMetrics err error @@ -61,14 +85,7 @@ func TestTopWorkersRunner_Success(t *testing.T) { WorkerPod: "counter-worker-pool-7b9f8-x123", SandboxClass: "gvisor", Labels: map[string]string{"ate.dev/worker-pool": "counter"}, - Status: &ateapipb.WorkerStatus{ - Assignment: &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{ - Atespace: "ate-demo-counter", - Name: "my-counter-1", - }, - }, - }, + Status: &ateapipb.WorkerStatus{Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}, Allocated: &ateapipb.WorkerResources{Actors: 1}}}, }, { WorkerNamespace: "ate-demo-counter", @@ -124,9 +141,9 @@ func TestTopWorkersRunner_Success(t *testing.T) { t.Fatalf("Run() unexpected error: %v", err) } - expected := `NAME POOL CLASS STATUS ASSIGNED ACTOR CPU(CORES) MEMORY(bytes) -counter-worker-pool-7b9f8-x123 counter gvisor ASSIGNED ate-demo-counter/my-counter-1 342m 412Mi -counter-worker-pool-7b9f8-y456 counter microvm FREE 2m 64Mi + expected := `NAME POOL CLASS STATUS CPU(CORES) MEMORY(bytes) +counter-worker-pool-7b9f8-x123 counter gvisor ASSIGNED(1/1) 342m 412Mi +counter-worker-pool-7b9f8-y456 counter microvm FREE 2m 64Mi ` if diff := cmp.Diff(expected, buf.String()); diff != "" { t.Errorf("output mismatch (-want +got):\n%s", diff) @@ -161,8 +178,8 @@ func TestTopWorkersRunner_FilterNamespace(t *testing.T) { t.Fatalf("Run() unexpected error: %v", err) } - expected := `NAME POOL CLASS STATUS ASSIGNED ACTOR CPU(CORES) MEMORY(bytes) -pod-1 pool-1 gvisor FREE metrics unavailable metrics unavailable + expected := `NAME POOL CLASS STATUS CPU(CORES) MEMORY(bytes) +pod-1 pool-1 gvisor FREE metrics unavailable metrics unavailable ` if diff := cmp.Diff(expected, buf.String()); diff != "" { t.Errorf("output mismatch (-want +got):\n%s", diff) @@ -172,31 +189,29 @@ pod-1 pool-1 gvisor FREE metrics unavailable metric func TestTopWorkersRunner_FilterAtespace(t *testing.T) { workers := []*ateapipb.Worker{ { + Metadata: &ateapipb.ResourceMetadata{Name: "worker-1"}, WorkerNamespace: "ns-1", WorkerPool: "pool-1", WorkerPod: "pod-1", SandboxClass: "microvm", - Status: &ateapipb.WorkerStatus{ - Assignment: &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: "space-a", Name: "actor-a"}, - }, - }, + Status: &ateapipb.WorkerStatus{Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}, Allocated: &ateapipb.WorkerResources{Actors: 1}}}, }, { + Metadata: &ateapipb.ResourceMetadata{Name: "worker-2"}, WorkerNamespace: "ns-1", WorkerPool: "pool-1", WorkerPod: "pod-2", - Status: &ateapipb.WorkerStatus{ - Assignment: &ateapipb.ActorAssignment{ - Actor: &ateapipb.ObjectRef{Atespace: "space-b", Name: "actor-b"}, - }, - }, + Status: &ateapipb.WorkerStatus{Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}, Allocated: &ateapipb.WorkerResources{Actors: 1}}}, }, } var buf bytes.Buffer runner := &TopWorkersRunner{ - workerLister: &mockWorkerLister{workers: workers}, + workerLister: &mockWorkerLister{workers: workers}, + actorLister: &mockActorLister{byAtespace: map[string][]*ateapipb.Actor{ + "space-a": {actorOn("space-a", "actor-a", "worker-1")}, + "space-b": {actorOn("space-b", "actor-b", "worker-2")}, + }}, podMetricsLister: &mockPodMetricsLister{metrics: nil}, atespace: "space-a", outputFmt: "table", @@ -207,8 +222,8 @@ func TestTopWorkersRunner_FilterAtespace(t *testing.T) { t.Fatalf("Run() unexpected error: %v", err) } - expected := `NAME POOL CLASS STATUS ASSIGNED ACTOR CPU(CORES) MEMORY(bytes) -pod-1 pool-1 microvm ASSIGNED space-a/actor-a metrics unavailable metrics unavailable + expected := `NAME POOL CLASS STATUS CPU(CORES) MEMORY(bytes) +pod-1 pool-1 microvm ASSIGNED(1/1) metrics unavailable metrics unavailable ` if diff := cmp.Diff(expected, buf.String()); diff != "" { t.Errorf("output mismatch (-want +got):\n%s", diff) @@ -245,8 +260,8 @@ func TestTopWorkersRunner_FilterSelector(t *testing.T) { t.Fatalf("Run() unexpected error: %v", err) } - expected := `NAME POOL CLASS STATUS ASSIGNED ACTOR CPU(CORES) MEMORY(bytes) -pod-1 counter gvisor FREE metrics unavailable metrics unavailable + expected := `NAME POOL CLASS STATUS CPU(CORES) MEMORY(bytes) +pod-1 counter gvisor FREE metrics unavailable metrics unavailable ` if diff := cmp.Diff(expected, buf.String()); diff != "" { t.Errorf("output mismatch (-want +got):\n%s", diff) @@ -274,8 +289,8 @@ func TestTopWorkersRunner_MetricsUnavailable(t *testing.T) { t.Fatalf("Run() unexpected error: %v", err) } - expected := `NAME POOL CLASS STATUS ASSIGNED ACTOR CPU(CORES) MEMORY(bytes) -pod-1 pool-1 FREE metrics unavailable metrics unavailable + expected := `NAME POOL CLASS STATUS CPU(CORES) MEMORY(bytes) +pod-1 pool-1 FREE metrics unavailable metrics unavailable ` if diff := cmp.Diff(expected, buf.String()); diff != "" { t.Errorf("output mismatch (-want +got):\n%s", diff) diff --git a/cmd/kubectl-ate/internal/cmd/workers.go b/cmd/kubectl-ate/internal/cmd/workers.go index 7a08538b99..47550f0e1b 100644 --- a/cmd/kubectl-ate/internal/cmd/workers.go +++ b/cmd/kubectl-ate/internal/cmd/workers.go @@ -28,6 +28,39 @@ type WorkerLister interface { ListWorkers(ctx context.Context, req *ateapipb.ListWorkersRequest, opts ...grpc.CallOption) (*ateapipb.ListWorkersResponse, error) } +// ActorLister abstracts ListActors RPC calls, which is how the worker commands +// resolve an atespace filter. +type ActorLister interface { + ListActors(ctx context.Context, req *ateapipb.ListActorsRequest, opts ...grpc.CallOption) (*ateapipb.ListActorsResponse, error) +} + +// workersHostingAtespace names the Workers hosting an Actor in atespace. Asked +// of the Actors, because a Worker listing reports how full each Worker is, not +// which Actors it holds. +func workersHostingAtespace(ctx context.Context, lister ActorLister, atespace string) (map[string]bool, error) { + hosting := map[string]bool{} + pageToken := "" + for { + resp, err := lister.ListActors(ctx, &ateapipb.ListActorsRequest{ + PageSize: 1000, + PageToken: pageToken, + Atespace: atespace, + }) + if err != nil { + return nil, fmt.Errorf("failed to list actors in atespace %q: %w", atespace, err) + } + for _, actor := range resp.GetActors() { + if name := actor.GetStatus().GetWorkerAssignment().GetWorker().GetName(); name != "" { + hosting[name] = true + } + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + return hosting, nil + } + } +} + // listAllWorkers pages through ListWorkers and returns all workers. func listAllWorkers(ctx context.Context, lister WorkerLister) ([]*ateapipb.Worker, error) { var workers []*ateapipb.Worker @@ -52,7 +85,7 @@ func listAllWorkers(ctx context.Context, lister WorkerLister) ([]*ateapipb.Worke // filterWorkers filters workers by Kubernetes namespace, assigned-actor // atespace, worker pool label selector, and sandbox class. Empty values match // everything; an atespace filter only matches workers with an assigned actor. -func filterWorkers(workers []*ateapipb.Worker, namespace, atespace, selector, sandboxClass string) ([]*ateapipb.Worker, error) { +func filterWorkers(ctx context.Context, actors ActorLister, workers []*ateapipb.Worker, namespace, atespace, selector, sandboxClass string) ([]*ateapipb.Worker, error) { var labelSel labels.Selector if selector != "" { var err error @@ -61,13 +94,22 @@ func filterWorkers(workers []*ateapipb.Worker, namespace, atespace, selector, sa return nil, fmt.Errorf("invalid label selector %q: %w", selector, err) } } + // A worker matches an atespace filter if any actor it hosts is in that + // atespace; an idle worker hosts none and so matches nothing. + var hostingAtespace map[string]bool + if atespace != "" { + var err error + if hostingAtespace, err = workersHostingAtespace(ctx, actors, atespace); err != nil { + return nil, err + } + } var filtered []*ateapipb.Worker for _, w := range workers { if namespace != "" && w.GetWorkerNamespace() != namespace { continue } - if atespace != "" && w.GetStatus().GetAssignment().GetActor().GetAtespace() != atespace { + if hostingAtespace != nil && !hostingAtespace[w.GetMetadata().GetName()] { continue } if labelSel != nil && !labelSel.Matches(labels.Set(w.GetLabels())) { diff --git a/cmd/kubectl-ate/internal/printer/printer.go b/cmd/kubectl-ate/internal/printer/printer.go index 35f63aafa7..ffaef5943e 100644 --- a/cmd/kubectl-ate/internal/printer/printer.go +++ b/cmd/kubectl-ate/internal/printer/printer.go @@ -102,6 +102,17 @@ func PrintWorkers(workers []*ateapipb.Worker, format string) error { return PrintWorkersTo(os.Stdout, workers, format) } +// WorkerOccupancy is how full a Worker is, as a count against its limit. A +// count rather than the Actors themselves: a listing does not carry them, and +// naming them all would be unreadable long before a Worker is full. +func WorkerOccupancy(worker *ateapipb.Worker) string { + hosted := worker.GetStatus().GetAllocation().GetAllocated().GetActors() + if hosted == 0 { + return "FREE" + } + return fmt.Sprintf("ASSIGNED(%d/%d)", hosted, worker.GetStatus().GetAllocation().GetCapacity().GetActors()) +} + func sortWorkers(workers []*ateapipb.Worker) { slices.SortFunc(workers, func(a, b *ateapipb.Worker) int { if c := cmp.Compare(a.GetWorkerNamespace(), b.GetWorkerNamespace()); c != 0 { @@ -122,23 +133,11 @@ func PrintWorkersTo(out io.Writer, workers []*ateapipb.Worker, format string) er return printProto(out, &ateapipb.ListWorkersResponse{Workers: workers}, format) case "table": w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0) - fmt.Fprintln(w, "NAMESPACE\tPOOL\tCLASS\tPOD\tSTATUS\tASSIGNED ACTOR") + fmt.Fprintln(w, "NAMESPACE\tPOOL\tCLASS\tPOD\tSTATUS") for _, worker := range workers { - ns := worker.GetWorkerNamespace() - pool := worker.GetWorkerPool() - class := worker.GetSandboxClass() - pod := worker.GetWorkerPod() - - status := "FREE" - assignedActor := "" - if wass := worker.GetStatus().GetAssignment(); wass != nil { - status = "ASSIGNED" - ref := wass.GetActorTemplateRef() - assignedActor = fmt.Sprintf("%s/%s/%s/%s", - ref.GetAtespace(), ref.GetName(), wass.GetActor().GetAtespace(), wass.GetActor().GetName()) - } - - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", ns, pool, class, pod, status, assignedActor) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", + worker.GetWorkerNamespace(), worker.GetWorkerPool(), worker.GetSandboxClass(), + worker.GetWorkerPod(), WorkerOccupancy(worker)) } return w.Flush() default: @@ -148,14 +147,13 @@ func PrintWorkersTo(out io.Writer, workers []*ateapipb.Worker, format string) er // WorkerTopItem represents real-time hardware resource utilization for a worker pod. type WorkerTopItem struct { - Pod string `json:"pod" yaml:"pod"` - Pool string `json:"pool" yaml:"pool"` - Class string `json:"class,omitempty" yaml:"class,omitempty"` - Status string `json:"status" yaml:"status"` - AssignedActor string `json:"assignedActor" yaml:"assignedActor"` - CPU string `json:"cpu" yaml:"cpu"` - Memory string `json:"memory" yaml:"memory"` - Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` + Pod string `json:"pod" yaml:"pod"` + Pool string `json:"pool" yaml:"pool"` + Class string `json:"class,omitempty" yaml:"class,omitempty"` + Status string `json:"status" yaml:"status"` + CPU string `json:"cpu" yaml:"cpu"` + Memory string `json:"memory" yaml:"memory"` + Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` } // WorkerTopList wraps worker top items for JSON/YAML output. @@ -198,10 +196,10 @@ func PrintWorkerTopTo(out io.Writer, items []*WorkerTopItem, format string) erro // PrintWorkerTopTable prints worker top items as a formatted table. func PrintWorkerTopTable(out io.Writer, items []*WorkerTopItem) error { w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0) - fmt.Fprintln(w, "NAME\tPOOL\tCLASS\tSTATUS\tASSIGNED ACTOR\tCPU(CORES)\tMEMORY(bytes)") + fmt.Fprintln(w, "NAME\tPOOL\tCLASS\tSTATUS\tCPU(CORES)\tMEMORY(bytes)") for _, item := range items { - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", - item.Pod, item.Pool, item.Class, item.Status, item.AssignedActor, item.CPU, item.Memory) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", + item.Pod, item.Pool, item.Class, item.Status, item.CPU, item.Memory) } return w.Flush() } diff --git a/cmd/kubectl-ate/internal/printer/printer_test.go b/cmd/kubectl-ate/internal/printer/printer_test.go index 4ddb94094b..d9067d8d1c 100644 --- a/cmd/kubectl-ate/internal/printer/printer_test.go +++ b/cmd/kubectl-ate/internal/printer/printer_test.go @@ -241,18 +241,7 @@ func TestPrintWorkersTo_Table(t *testing.T) { WorkerPool: "pool-1", WorkerPod: "pod-1", SandboxClass: "gvisor", - Status: &ateapipb.WorkerStatus{ - Assignment: &ateapipb.ActorAssignment{ - ActorTemplateRef: &ateapipb.ObjectRef{ - Atespace: "default", - Name: "template-1", - }, - Actor: &ateapipb.ObjectRef{ - Atespace: "space-1", - Name: "id-1", - }, - }, - }, + Status: &ateapipb.WorkerStatus{Allocation: &ateapipb.WorkerAllocation{Capacity: &ateapipb.WorkerResources{Actors: 1}, Allocated: &ateapipb.WorkerResources{Actors: 1}}}, }, } @@ -261,14 +250,17 @@ func TestPrintWorkersTo_Table(t *testing.T) { } output := buf.String() - expected := `NAMESPACE POOL CLASS POD STATUS ASSIGNED ACTOR -default pool-1 gvisor pod-1 ASSIGNED default/template-1/space-1/id-1 + expected := `NAMESPACE POOL CLASS POD STATUS +default pool-1 gvisor pod-1 ASSIGNED(1/1) ` if diff := cmp.Diff(expected, output); diff != "" { t.Errorf("output mismatch (-want +got):\n%s", diff) } } +// A worker assigned to an actor created from a substrate ActorTemplate +// carries only ActorTemplateRef; the printer must not dereference the legacy +// CRD ref (regression test for a nil-pointer panic). func TestPrintWorkersTo_Table_Free(t *testing.T) { var buf bytes.Buffer workers := []*ateapipb.Worker{ @@ -284,8 +276,8 @@ func TestPrintWorkersTo_Table_Free(t *testing.T) { } output := buf.String() - expected := `NAMESPACE POOL CLASS POD STATUS ASSIGNED ACTOR -default pool-1 pod-1 FREE + expected := `NAMESPACE POOL CLASS POD STATUS +default pool-1 pod-1 FREE ` if diff := cmp.Diff(expected, output); diff != "" { t.Errorf("output mismatch (-want +got):\n%s", diff) @@ -316,10 +308,10 @@ func TestPrintWorkersTo_Table_Sorted(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - expected := `NAMESPACE POOL CLASS POD STATUS ASSIGNED ACTOR -default pool-1 pod-a FREE -default pool-1 pod-z FREE -other pool-2 pod-1 FREE + expected := `NAMESPACE POOL CLASS POD STATUS +default pool-1 pod-a FREE +default pool-1 pod-z FREE +other pool-2 pod-1 FREE ` if diff := cmp.Diff(expected, buf.String()); diff != "" { t.Errorf("output mismatch (-want +got):\n%s", diff) @@ -564,24 +556,22 @@ func TestPrintWorkerTopTo_Table(t *testing.T) { var buf bytes.Buffer items := []*WorkerTopItem{ { - Pod: "counter-worker-pool-7b9f8-x123", - Pool: "counter", - Class: "gvisor", - Status: "ASSIGNED", - AssignedActor: "default/counter-template/ate-demo-counter/my-counter-1", - CPU: "342m", - Memory: "412Mi", - Namespace: "ate-demo-counter", + Pod: "counter-worker-pool-7b9f8-x123", + Pool: "counter", + Class: "gvisor", + Status: "ASSIGNED", + CPU: "342m", + Memory: "412Mi", + Namespace: "ate-demo-counter", }, { - Pod: "counter-worker-pool-7b9f8-y456", - Pool: "counter", - Class: "microvm", - Status: "FREE", - AssignedActor: "", - CPU: "2m", - Memory: "64Mi", - Namespace: "ate-demo-counter", + Pod: "counter-worker-pool-7b9f8-y456", + Pool: "counter", + Class: "microvm", + Status: "FREE", + CPU: "2m", + Memory: "64Mi", + Namespace: "ate-demo-counter", }, } @@ -590,9 +580,9 @@ func TestPrintWorkerTopTo_Table(t *testing.T) { } output := buf.String() - expected := `NAME POOL CLASS STATUS ASSIGNED ACTOR CPU(CORES) MEMORY(bytes) -counter-worker-pool-7b9f8-x123 counter gvisor ASSIGNED default/counter-template/ate-demo-counter/my-counter-1 342m 412Mi -counter-worker-pool-7b9f8-y456 counter microvm FREE 2m 64Mi + expected := `NAME POOL CLASS STATUS CPU(CORES) MEMORY(bytes) +counter-worker-pool-7b9f8-x123 counter gvisor ASSIGNED 342m 412Mi +counter-worker-pool-7b9f8-y456 counter microvm FREE 2m 64Mi ` if diff := cmp.Diff(expected, output); diff != "" { t.Errorf("output mismatch (-want +got):\n%s", diff) @@ -603,12 +593,11 @@ func TestPrintWorkerTopTo_JSON(t *testing.T) { var buf bytes.Buffer items := []*WorkerTopItem{ { - Pod: "worker-1", - Pool: "pool-1", - Status: "ASSIGNED", - AssignedActor: "default/template-1/space-1/actor-1", - CPU: "100m", - Memory: "128Mi", + Pod: "worker-1", + Pool: "pool-1", + Status: "ASSIGNED", + CPU: "100m", + Memory: "128Mi", }, } @@ -623,7 +612,6 @@ func TestPrintWorkerTopTo_JSON(t *testing.T) { "pod": "worker-1", "pool": "pool-1", "status": "ASSIGNED", - "assignedActor": "default/template-1/space-1/actor-1", "cpu": "100m", "memory": "128Mi" } @@ -639,12 +627,11 @@ func TestPrintWorkerTopTo_YAML(t *testing.T) { var buf bytes.Buffer items := []*WorkerTopItem{ { - Pod: "worker-1", - Pool: "pool-1", - Status: "ASSIGNED", - AssignedActor: "default/template-1/space-1/actor-1", - CPU: "100m", - Memory: "128Mi", + Pod: "worker-1", + Pool: "pool-1", + Status: "ASSIGNED", + CPU: "100m", + Memory: "128Mi", }, } @@ -654,8 +641,7 @@ func TestPrintWorkerTopTo_YAML(t *testing.T) { output := buf.String() expected := `workers: -- assignedActor: default/template-1/space-1/actor-1 - cpu: 100m +- cpu: 100m memory: 128Mi pod: worker-1 pool: pool-1 diff --git a/demos/claude-code-multiplex/ui/server.go b/demos/claude-code-multiplex/ui/server.go index c9a28cbffb..2ecf3599f8 100644 --- a/demos/claude-code-multiplex/ui/server.go +++ b/demos/claude-code-multiplex/ui/server.go @@ -220,7 +220,7 @@ func actorStateString(s ateapipb.ActorState) string { // The UI's badgeFor() treats "running" as green; "idle" falls through // to the neutral badge, which is the right visual treatment. func workerPhase(w *ateapipb.Worker) string { - if w.GetStatus().GetAssignment().GetActor().GetName() != "" { + if w.GetStatus().GetAllocation().GetAllocated().GetActors() > 0 { return "Running" } return "Idle" @@ -372,18 +372,9 @@ func handlePods(w http.ResponseWriter, r *http.Request) { } pods := make([]podSummary, 0, len(resp.GetWorkers())) for _, wk := range resp.GetWorkers() { - // Filter to the demo namespace when set — workers may live - // in their own pool namespace (worker_namespace) so we - // compare against actor_namespace too. - if ref := wk.GetStatus().GetAssignment().GetActorTemplateRef(); ref != nil { - if ns, wkns := namespace, ref.GetAtespace(); ns != "" && wkns != "" && wkns != ns { - continue - } - } - ready := false - if wk.GetStatus().GetAssignment().GetActor().GetName() != "" { - ready = true - } + // No demo-namespace filter: a Worker listing does not name the Actors + // it holds, so there is no template namespace to compare against. + ready := wk.GetStatus().GetAllocation().GetAllocated().GetActors() > 0 pods = append(pods, podSummary{ Name: wk.GetWorkerPod(), Node: wk.GetWorkerPool(), // closest semantic analog diff --git a/hack/update/codegen.sh b/hack/update/codegen.sh index 99055e2e0e..48ced4316b 100755 --- a/hack/update/codegen.sh +++ b/hack/update/codegen.sh @@ -62,6 +62,7 @@ function codegen::protobuf() { ( cd "${dir}" || exit 1 "${ROOT}"/hack/protoc.sh \ + -I "${ROOT}" -I . \ --plugin=protoc-gen-go="${protoc_gen_go}" \ --plugin=protoc-gen-go-grpc="${protoc_gen_go_rpc}" \ --go_out=paths=source_relative:. \ diff --git a/internal/ateletdial/ateletdial.go b/internal/ateletdial/ateletdial.go new file mode 100644 index 0000000000..dc51d21c00 --- /dev/null +++ b/internal/ateletdial/ateletdial.go @@ -0,0 +1,99 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ateletdial connects a worker Pod to the atelet on its own node over +// the node-local socket, authenticating both ends by Pod certificate. +package ateletdial + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "net" + "net/url" + "os" + "path" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/substratex509" +) + +// TLSConfig authenticates this worker to atelet with its Pod certificate, and +// accepts only the atelet on this worker's own node. +func TLSConfig(credentialBundlePath, trustBundlePath string) (*tls.Config, error) { + if credentialBundlePath == "" || trustBundlePath == "" { + return nil, fmt.Errorf("worker credentials and trust bundle are required") + } + localCert, err := credbundle.Parse(credentialBundlePath) + if err != nil { + return nil, fmt.Errorf("load worker identity: %w", err) + } + localIdentity, err := substratex509.PodIdentityFromCertificate(localCert.Leaf) + if err != nil || localIdentity == nil { + return nil, fmt.Errorf("worker certificate has no valid Pod identity") + } + trustPEM, err := os.ReadFile(trustBundlePath) + if err != nil { + return nil, fmt.Errorf("read atelet trust bundle: %w", err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(trustPEM) { + return nil, fmt.Errorf("atelet trust bundle contains no certificates") + } + expectedURI := (&url.URL{Scheme: "spiffe", Host: "cluster.local", Path: path.Join("ns", "ate-system", "sa", "atelet")}).String() + return &tls.Config{ + MinVersion: tls.VersionTLS13, + InsecureSkipVerify: true, // Verification below supports SPIFFE Pod certificates without a DNS name. + GetClientCertificate: credbundle.ClientLoader(credentialBundlePath), + VerifyConnection: func(state tls.ConnectionState) error { + // Verify both the normal server-auth chain and the identities that DNS + // verification cannot express: atelet's SPIFFE ID and exact node + // incarnation. This is why InsecureSkipVerify is set above. + if len(state.PeerCertificates) == 0 { + return fmt.Errorf("atelet certificate is required") + } + intermediates := x509.NewCertPool() + for _, cert := range state.PeerCertificates[1:] { + intermediates.AddCert(cert) + } + if _, err := state.PeerCertificates[0].Verify(x509.VerifyOptions{Roots: roots, Intermediates: intermediates, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}); err != nil { + return fmt.Errorf("verify atelet certificate: %w", err) + } + leaf := state.PeerCertificates[0] + if len(leaf.URIs) != 1 || leaf.URIs[0].String() != expectedURI { + return fmt.Errorf("node-local peer is not atelet") + } + identity, err := substratex509.PodIdentityFromCertificate(leaf) + if err != nil || identity == nil || identity.NodeName != localIdentity.NodeName || identity.NodeUID != localIdentity.NodeUID { + return fmt.Errorf("atelet is not on worker node %q (%s)", localIdentity.NodeName, localIdentity.NodeUID) + } + return nil + }, + }, nil +} + +// Dial opens a connection to the atelet socket. The caller closes it; a fresh +// connection picks up rotated worker credentials and re-verifies atelet. +func Dial(socketPath string, tlsConfig *tls.Config) (*grpc.ClientConn, error) { + return grpc.NewClient("passthrough:///atelet", + grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", socketPath) + }), + ) +} diff --git a/internal/ateomcapacity/ateomcapacity.go b/internal/ateomcapacity/ateomcapacity.go new file mode 100644 index 0000000000..731304d053 --- /dev/null +++ b/internal/ateomcapacity/ateomcapacity.go @@ -0,0 +1,156 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ateomcapacity reports what an ateom can supply to the actors it +// hosts. Both ateoms answer GetCapacity from here so they answer it alike. +package ateomcapacity + +import ( + "context" + "crypto/tls" + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/agent-substrate/substrate/internal/ateletdial" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + + "github.com/agent-substrate/substrate/internal/proto/ateletpb" +) + +// Files the atecontroller projects into the ateom container from the downward +// API, in milli-cores and bytes. +const ( + CapacityMountPath = "/run/ateom-capacity" + CPULimitFile = "cpu_milli" + MemoryLimitFile = "memory_bytes" +) + +// actorsPerAteom is how many actors an ateom hosts at once. One, today. +const actorsPerAteom = 1 + +const ( + reportTimeout = 10 * time.Second + initialReportBackoff = 500 * time.Millisecond + maxReportBackoff = 30 * time.Second +) + +// FromFiles reads the ateom's compute limits from its downward API volume, as +// the report it sends to the node-local atelet. +// +// A limit that is missing or unparseable is reported as zero, which the control +// plane reads as none: better to place nothing on a worker that cannot say what +// it has than to invent a number for it. +// +// TODO: Watch the projected files and report changes. For now we do not support +// in-place Pod vertical scaling (IPPR); capacity is read once at startup. +// NOTE: Please do not implement this yet. IPPR needs more general consideration. +func FromFiles() *ateletpb.SetWorkerCapacityRequest { + return fromDir(CapacityMountPath) +} + +func fromDir(dir string) *ateletpb.SetWorkerCapacityRequest { + return &ateletpb.SetWorkerCapacityRequest{ + Capacity: &ateapipb.WorkerResources{ + Actors: actorsPerAteom, + // A limit read as zero is left out, which the control plane reads + // as none of that dimension. + Resources: resources.CPUMemory( + readLimit(filepath.Join(dir, CPULimitFile)), + readLimit(filepath.Join(dir, MemoryLimitFile)), + ), + }, + } +} + +func readLimit(path string) int64 { + contents, err := os.ReadFile(path) + if err != nil { + slog.Warn("Ignoring unreadable capacity limit", slog.String("path", path), slog.Any("err", err)) + return 0 + } + raw := strings.TrimSpace(string(contents)) + value, err := strconv.ParseInt(raw, 10, 64) + if err != nil || value < 0 { + slog.Warn("Ignoring unusable capacity limit", slog.String("path", path), slog.String("value", raw)) + return 0 + } + return value +} + +// ReportConfig is what an ateom needs to reach the atelet on its node. +type ReportConfig struct { + SocketPath string + CredentialBundlePath string + TrustBundlePath string +} + +// Report tells the node-local atelet what this ateom can supply, retrying +// until it is accepted or ctx ends. +// +// Retrying is what makes a single report durable: atelet only accepts once the +// control plane has recorded it, and the Worker record may not exist yet when +// an ateom first comes up. Nothing else reports this, so giving up would leave +// the Worker holding no capacity and hosting nothing. +func Report(ctx context.Context, cfg ReportConfig) error { + tlsConfig, err := ateletdial.TLSConfig(cfg.CredentialBundlePath, cfg.TrustBundlePath) + if err != nil { + return fmt.Errorf("capacity report: %w", err) + } + capacity := FromFiles() + err = retryReport(ctx, func() error { + return reportOnce(ctx, cfg.SocketPath, tlsConfig, capacity) + }, initialReportBackoff) + if err != nil { + return err + } + slog.InfoContext(ctx, "Reported worker capacity", slog.Any("capacity", capacity.GetCapacity())) + return nil +} + +// retryReport calls send until it succeeds or ctx ends, backing off between +// attempts. send is a parameter so the loop can be exercised without a socket +// or certificates. +func retryReport(ctx context.Context, send func() error, backoff time.Duration) error { + for { + err := send() + if err == nil { + return nil + } + slog.WarnContext(ctx, "Retrying worker capacity report", slog.Duration("in", backoff), slog.Any("err", err)) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(backoff): + } + backoff = min(backoff*2, maxReportBackoff) + } +} + +func reportOnce(ctx context.Context, socketPath string, tlsConfig *tls.Config, capacity *ateletpb.SetWorkerCapacityRequest) error { + conn, err := ateletdial.Dial(socketPath, tlsConfig) + if err != nil { + return err + } + defer conn.Close() + callCtx, cancel := context.WithTimeout(ctx, reportTimeout) + defer cancel() + _, err = ateletpb.NewWorkerCapacityClient(conn).SetWorkerCapacity(callCtx, capacity) + return err +} diff --git a/internal/ateomcapacity/ateomcapacity_test.go b/internal/ateomcapacity/ateomcapacity_test.go new file mode 100644 index 0000000000..ffe00a174e --- /dev/null +++ b/internal/ateomcapacity/ateomcapacity_test.go @@ -0,0 +1,125 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ateomcapacity + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/testing/protocmp" + + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +func TestFromFiles(t *testing.T) { + for _, tc := range []struct { + name string + cpu string + memory string + want *ateapipb.Resources + }{ + {name: "limits set", cpu: "2000", memory: "4294967296", want: resources.CPUMemory(2000, 4294967296)}, + {name: "unparseable is none", cpu: "2Gi", memory: "", want: nil}, + {name: "negative is none", cpu: "-1", memory: "-1", want: nil}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, CPULimitFile), []byte(tc.cpu), 0o600); err != nil { + t.Fatalf("writing CPU limit: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, MemoryLimitFile), []byte(tc.memory), 0o600); err != nil { + t.Fatalf("writing memory limit: %v", err) + } + + got := fromDir(dir).GetCapacity() + if got.GetActors() != actorsPerAteom { + t.Errorf("actors = %d, want %d", got.GetActors(), actorsPerAteom) + } + if diff := cmp.Diff(tc.want, got.GetResources(), protocmp.Transform()); diff != "" { + t.Errorf("reported resources mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestFromFilesMissing(t *testing.T) { + got := fromDir(t.TempDir()).GetCapacity() + if got.GetResources() != nil { + t.Errorf("unset environment reported %v, want no compute", got.GetResources()) + } + if got.GetActors() != actorsPerAteom { + t.Errorf("actors = %d, want %d", got.GetActors(), actorsPerAteom) + } +} + +// reportSeam swaps the one-shot call out so the retry loop can be exercised +// without a socket or certificates. +func TestReportRetriesUntilAccepted(t *testing.T) { + attempts := 0 + send := func() error { + attempts++ + if attempts < 3 { + return errors.New("worker record does not exist yet") + } + return nil + } + if err := retryReport(context.Background(), send, time.Millisecond); err != nil { + t.Fatalf("retryReport() failed: %v", err) + } + if attempts != 3 { + t.Errorf("gave up after %d attempts, want 3", attempts) + } +} + +func TestReportStopsWhenContextEnds(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + attempts := 0 + send := func() error { + attempts++ + if attempts == 2 { + cancel() + } + return errors.New("still failing") + } + if err := retryReport(ctx, send, time.Millisecond); !errors.Is(err, context.Canceled) { + t.Errorf("retryReport() = %v, want context.Canceled", err) + } +} + +// A misconfiguration must surface rather than spin in the retry loop: the +// caller exits on it, and retrying forever would leave the worker idle and +// silent instead. +func TestReportFailsFastOnBadCredentials(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + err := Report(ctx, ReportConfig{ + SocketPath: filepath.Join(t.TempDir(), "atelet.sock"), + CredentialBundlePath: filepath.Join(t.TempDir(), "does-not-exist.pem"), + TrustBundlePath: filepath.Join(t.TempDir(), "also-missing.pem"), + }) + if err == nil { + t.Fatal("Report() with unreadable credentials succeeded, want an error the caller can exit on") + } + if errors.Is(err, context.DeadlineExceeded) { + t.Errorf("Report() retried a permanent failure until the deadline: %v", err) + } +} diff --git a/internal/atunnel/credential.go b/internal/atunnel/credential.go index 65090af769..501ce75433 100644 --- a/internal/atunnel/credential.go +++ b/internal/atunnel/credential.go @@ -22,19 +22,13 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "net" - "net/url" - "os" - "path" "slices" "sync" "time" - "github.com/agent-substrate/substrate/internal/credbundle" + "github.com/agent-substrate/substrate/internal/ateletdial" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/substratex509" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) // BrokerCertificateSource owns atunnel's actor private key and obtains the @@ -70,56 +64,14 @@ func NewBrokerCertificateSource(cfg BrokerConfig) (*BrokerCertificateSource, err if cfg.SocketPath == "" || cfg.CredentialBundlePath == "" || cfg.TrustBundlePath == "" || cfg.ExpectedActorUID == "" { return nil, fmt.Errorf("atunnel: credential broker socket, credentials, trust bundle, and expected actor UID are required") } - localCert, err := credbundle.Parse(cfg.CredentialBundlePath) + tlsConfig, err := ateletdial.TLSConfig(cfg.CredentialBundlePath, cfg.TrustBundlePath) if err != nil { - return nil, fmt.Errorf("atunnel: load worker identity: %w", err) - } - localIdentity, err := substratex509.PodIdentityFromCertificate(localCert.Leaf) - if err != nil || localIdentity == nil { - return nil, fmt.Errorf("atunnel: worker certificate has no valid Pod identity") - } - trustPEM, err := os.ReadFile(cfg.TrustBundlePath) - if err != nil { - return nil, fmt.Errorf("atunnel: read credential broker trust bundle: %w", err) - } - roots := x509.NewCertPool() - if !roots.AppendCertsFromPEM(trustPEM) { - return nil, fmt.Errorf("atunnel: credential broker trust bundle contains no certificates") + return nil, fmt.Errorf("atunnel: %w", err) } privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { return nil, fmt.Errorf("atunnel: generate actor private key: %w", err) } - expectedURI := (&url.URL{Scheme: "spiffe", Host: "cluster.local", Path: path.Join("ns", "ate-system", "sa", "atelet")}).String() - tlsConfig := &tls.Config{ - MinVersion: tls.VersionTLS13, - InsecureSkipVerify: true, // Verification below supports SPIFFE Pod certificates without a DNS name. - GetClientCertificate: credbundle.ClientLoader(cfg.CredentialBundlePath), - VerifyConnection: func(state tls.ConnectionState) error { - // Verify both the normal server-auth chain and the identities that DNS - // verification cannot express: atelet's SPIFFE ID and exact node - // incarnation. This is why InsecureSkipVerify is set above. - if len(state.PeerCertificates) == 0 { - return fmt.Errorf("credential broker certificate is required") - } - intermediates := x509.NewCertPool() - for _, cert := range state.PeerCertificates[1:] { - intermediates.AddCert(cert) - } - if _, err := state.PeerCertificates[0].Verify(x509.VerifyOptions{Roots: roots, Intermediates: intermediates, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}); err != nil { - return fmt.Errorf("verify credential broker certificate: %w", err) - } - leaf := state.PeerCertificates[0] - if len(leaf.URIs) != 1 || leaf.URIs[0].String() != expectedURI { - return fmt.Errorf("credential broker is not atelet") - } - identity, err := substratex509.PodIdentityFromCertificate(leaf) - if err != nil || identity == nil || identity.NodeName != localIdentity.NodeName || identity.NodeUID != localIdentity.NodeUID { - return fmt.Errorf("credential broker is not on worker node %q (%s)", localIdentity.NodeName, localIdentity.NodeUID) - } - return nil - }, - } return &BrokerCertificateSource{socketPath: cfg.SocketPath, expectedActorUID: cfg.ExpectedActorUID, tlsConfig: tlsConfig, privateKey: privateKey}, nil } @@ -133,12 +85,7 @@ func (s *BrokerCertificateSource) Mint(ctx context.Context) (time.Time, error) { } // A fresh connection picks up rotated worker credentials and forces atelet's // current certificate and node identity to be verified for every mint. - conn, err := grpc.NewClient("passthrough:///credential-broker", - grpc.WithTransportCredentials(credentials.NewTLS(s.tlsConfig)), - grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { - return (&net.Dialer{}).DialContext(ctx, "unix", s.socketPath) - }), - ) + conn, err := ateletdial.Dial(s.socketPath, s.tlsConfig) if err != nil { return time.Time{}, err } diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index a060b537cd..9bd0297ab6 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -21,6 +21,7 @@ package ateletpb import ( + ateapipb "github.com/agent-substrate/substrate/pkg/proto/ateapipb" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -204,6 +205,88 @@ func (SnapshotScope) EnumDescriptor() ([]byte, []int) { return file_atelet_proto_rawDescGZIP(), []int{2} } +type SetWorkerCapacityRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // What the worker can supply, in the same vocabulary the control plane + // records and an ActorTemplate asks in. + Capacity *ateapipb.WorkerResources `protobuf:"bytes,1,opt,name=capacity,proto3" json:"capacity,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetWorkerCapacityRequest) Reset() { + *x = SetWorkerCapacityRequest{} + mi := &file_atelet_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetWorkerCapacityRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetWorkerCapacityRequest) ProtoMessage() {} + +func (x *SetWorkerCapacityRequest) ProtoReflect() protoreflect.Message { + mi := &file_atelet_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetWorkerCapacityRequest.ProtoReflect.Descriptor instead. +func (*SetWorkerCapacityRequest) Descriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{0} +} + +func (x *SetWorkerCapacityRequest) GetCapacity() *ateapipb.WorkerResources { + if x != nil { + return x.Capacity + } + return nil +} + +type SetWorkerCapacityResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetWorkerCapacityResponse) Reset() { + *x = SetWorkerCapacityResponse{} + mi := &file_atelet_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetWorkerCapacityResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetWorkerCapacityResponse) ProtoMessage() {} + +func (x *SetWorkerCapacityResponse) ProtoReflect() protoreflect.Message { + mi := &file_atelet_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetWorkerCapacityResponse.ProtoReflect.Descriptor instead. +func (*SetWorkerCapacityResponse) Descriptor() ([]byte, []int) { + return file_atelet_proto_rawDescGZIP(), []int{1} +} + type MintActorCertificateRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // DER-encoded PKCS #10 certificate signing request. Atunnel retains the @@ -218,7 +301,7 @@ type MintActorCertificateRequest struct { func (x *MintActorCertificateRequest) Reset() { *x = MintActorCertificateRequest{} - mi := &file_atelet_proto_msgTypes[0] + mi := &file_atelet_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -230,7 +313,7 @@ func (x *MintActorCertificateRequest) String() string { func (*MintActorCertificateRequest) ProtoMessage() {} func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[0] + mi := &file_atelet_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -243,7 +326,7 @@ func (x *MintActorCertificateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateRequest.ProtoReflect.Descriptor instead. func (*MintActorCertificateRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{0} + return file_atelet_proto_rawDescGZIP(), []int{2} } func (x *MintActorCertificateRequest) GetCertificateSigningRequest() []byte { @@ -270,7 +353,7 @@ type MintActorCertificateResponse struct { func (x *MintActorCertificateResponse) Reset() { *x = MintActorCertificateResponse{} - mi := &file_atelet_proto_msgTypes[1] + mi := &file_atelet_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -282,7 +365,7 @@ func (x *MintActorCertificateResponse) String() string { func (*MintActorCertificateResponse) ProtoMessage() {} func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[1] + mi := &file_atelet_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -295,7 +378,7 @@ func (x *MintActorCertificateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintActorCertificateResponse.ProtoReflect.Descriptor instead. func (*MintActorCertificateResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{1} + return file_atelet_proto_rawDescGZIP(), []int{3} } func (x *MintActorCertificateResponse) GetActorCertificates() [][]byte { @@ -320,7 +403,7 @@ type TerminateRequest struct { func (x *TerminateRequest) Reset() { *x = TerminateRequest{} - mi := &file_atelet_proto_msgTypes[2] + mi := &file_atelet_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -332,7 +415,7 @@ func (x *TerminateRequest) String() string { func (*TerminateRequest) ProtoMessage() {} func (x *TerminateRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[2] + mi := &file_atelet_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -345,7 +428,7 @@ func (x *TerminateRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use TerminateRequest.ProtoReflect.Descriptor instead. func (*TerminateRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{2} + return file_atelet_proto_rawDescGZIP(), []int{4} } func (x *TerminateRequest) GetTargetAteomUid() string { @@ -405,7 +488,7 @@ type TerminateResponse struct { func (x *TerminateResponse) Reset() { *x = TerminateResponse{} - mi := &file_atelet_proto_msgTypes[3] + mi := &file_atelet_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -417,7 +500,7 @@ func (x *TerminateResponse) String() string { func (*TerminateResponse) ProtoMessage() {} func (x *TerminateResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[3] + mi := &file_atelet_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -430,7 +513,7 @@ func (x *TerminateResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use TerminateResponse.ProtoReflect.Descriptor instead. func (*TerminateResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{3} + return file_atelet_proto_rawDescGZIP(), []int{5} } type RunRequest struct { @@ -459,7 +542,7 @@ type RunRequest struct { func (x *RunRequest) Reset() { *x = RunRequest{} - mi := &file_atelet_proto_msgTypes[4] + mi := &file_atelet_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -471,7 +554,7 @@ func (x *RunRequest) String() string { func (*RunRequest) ProtoMessage() {} func (x *RunRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[4] + mi := &file_atelet_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -484,7 +567,7 @@ func (x *RunRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RunRequest.ProtoReflect.Descriptor instead. func (*RunRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{4} + return file_atelet_proto_rawDescGZIP(), []int{6} } func (x *RunRequest) GetTargetAteomUid() string { @@ -576,7 +659,7 @@ type EgressGateway struct { func (x *EgressGateway) Reset() { *x = EgressGateway{} - mi := &file_atelet_proto_msgTypes[5] + mi := &file_atelet_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -588,7 +671,7 @@ func (x *EgressGateway) String() string { func (*EgressGateway) ProtoMessage() {} func (x *EgressGateway) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[5] + mi := &file_atelet_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -601,7 +684,7 @@ func (x *EgressGateway) ProtoReflect() protoreflect.Message { // Deprecated: Use EgressGateway.ProtoReflect.Descriptor instead. func (*EgressGateway) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{5} + return file_atelet_proto_rawDescGZIP(), []int{7} } func (x *EgressGateway) GetAddress() string { @@ -625,7 +708,7 @@ type AssetFile struct { func (x *AssetFile) Reset() { *x = AssetFile{} - mi := &file_atelet_proto_msgTypes[6] + mi := &file_atelet_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -637,7 +720,7 @@ func (x *AssetFile) String() string { func (*AssetFile) ProtoMessage() {} func (x *AssetFile) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[6] + mi := &file_atelet_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -650,7 +733,7 @@ func (x *AssetFile) ProtoReflect() protoreflect.Message { // Deprecated: Use AssetFile.ProtoReflect.Descriptor instead. func (*AssetFile) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{6} + return file_atelet_proto_rawDescGZIP(), []int{8} } func (x *AssetFile) GetUrl() string { @@ -678,7 +761,7 @@ type ArchAssets struct { func (x *ArchAssets) Reset() { *x = ArchAssets{} - mi := &file_atelet_proto_msgTypes[7] + mi := &file_atelet_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -690,7 +773,7 @@ func (x *ArchAssets) String() string { func (*ArchAssets) ProtoMessage() {} func (x *ArchAssets) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[7] + mi := &file_atelet_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -703,7 +786,7 @@ func (x *ArchAssets) ProtoReflect() protoreflect.Message { // Deprecated: Use ArchAssets.ProtoReflect.Descriptor instead. func (*ArchAssets) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{7} + return file_atelet_proto_rawDescGZIP(), []int{9} } func (x *ArchAssets) GetFiles() map[string]*AssetFile { @@ -733,7 +816,7 @@ type SandboxAssets struct { func (x *SandboxAssets) Reset() { *x = SandboxAssets{} - mi := &file_atelet_proto_msgTypes[8] + mi := &file_atelet_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -745,7 +828,7 @@ func (x *SandboxAssets) String() string { func (*SandboxAssets) ProtoMessage() {} func (x *SandboxAssets) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[8] + mi := &file_atelet_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -758,7 +841,7 @@ func (x *SandboxAssets) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxAssets.ProtoReflect.Descriptor instead. func (*SandboxAssets) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{8} + return file_atelet_proto_rawDescGZIP(), []int{10} } func (x *SandboxAssets) GetSandboxClass() string { @@ -793,7 +876,7 @@ type WorkloadSpec struct { func (x *WorkloadSpec) Reset() { *x = WorkloadSpec{} - mi := &file_atelet_proto_msgTypes[9] + mi := &file_atelet_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -805,7 +888,7 @@ func (x *WorkloadSpec) String() string { func (*WorkloadSpec) ProtoMessage() {} func (x *WorkloadSpec) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[9] + mi := &file_atelet_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -818,7 +901,7 @@ func (x *WorkloadSpec) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkloadSpec.ProtoReflect.Descriptor instead. func (*WorkloadSpec) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{9} + return file_atelet_proto_rawDescGZIP(), []int{11} } func (x *WorkloadSpec) GetContainers() []*Container { @@ -843,7 +926,7 @@ type DurableDirVolume struct { func (x *DurableDirVolume) Reset() { *x = DurableDirVolume{} - mi := &file_atelet_proto_msgTypes[10] + mi := &file_atelet_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -855,7 +938,7 @@ func (x *DurableDirVolume) String() string { func (*DurableDirVolume) ProtoMessage() {} func (x *DurableDirVolume) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[10] + mi := &file_atelet_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -868,7 +951,7 @@ func (x *DurableDirVolume) ProtoReflect() protoreflect.Message { // Deprecated: Use DurableDirVolume.ProtoReflect.Descriptor instead. func (*DurableDirVolume) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{10} + return file_atelet_proto_rawDescGZIP(), []int{12} } type ExternalVolumeSource struct { @@ -882,7 +965,7 @@ type ExternalVolumeSource struct { func (x *ExternalVolumeSource) Reset() { *x = ExternalVolumeSource{} - mi := &file_atelet_proto_msgTypes[11] + mi := &file_atelet_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -894,7 +977,7 @@ func (x *ExternalVolumeSource) String() string { func (*ExternalVolumeSource) ProtoMessage() {} func (x *ExternalVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[11] + mi := &file_atelet_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -907,7 +990,7 @@ func (x *ExternalVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalVolumeSource.ProtoReflect.Descriptor instead. func (*ExternalVolumeSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{11} + return file_atelet_proto_rawDescGZIP(), []int{13} } func (x *ExternalVolumeSource) GetStorageVolumeId() string { @@ -940,7 +1023,7 @@ type ImageVolumeSource struct { func (x *ImageVolumeSource) Reset() { *x = ImageVolumeSource{} - mi := &file_atelet_proto_msgTypes[12] + mi := &file_atelet_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -952,7 +1035,7 @@ func (x *ImageVolumeSource) String() string { func (*ImageVolumeSource) ProtoMessage() {} func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[12] + mi := &file_atelet_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -965,7 +1048,7 @@ func (x *ImageVolumeSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ImageVolumeSource.ProtoReflect.Descriptor instead. func (*ImageVolumeSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{12} + return file_atelet_proto_rawDescGZIP(), []int{14} } func (x *ImageVolumeSource) GetReference() string { @@ -987,7 +1070,7 @@ type ActorMetadataItem struct { func (x *ActorMetadataItem) Reset() { *x = ActorMetadataItem{} - mi := &file_atelet_proto_msgTypes[13] + mi := &file_atelet_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -999,7 +1082,7 @@ func (x *ActorMetadataItem) String() string { func (*ActorMetadataItem) ProtoMessage() {} func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[13] + mi := &file_atelet_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1012,7 +1095,7 @@ func (x *ActorMetadataItem) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataItem.ProtoReflect.Descriptor instead. func (*ActorMetadataItem) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{13} + return file_atelet_proto_rawDescGZIP(), []int{15} } func (x *ActorMetadataItem) GetField() ActorMetadataField { @@ -1040,7 +1123,7 @@ type ActorMetadataDataSource struct { func (x *ActorMetadataDataSource) Reset() { *x = ActorMetadataDataSource{} - mi := &file_atelet_proto_msgTypes[14] + mi := &file_atelet_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1052,7 +1135,7 @@ func (x *ActorMetadataDataSource) String() string { func (*ActorMetadataDataSource) ProtoMessage() {} func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[14] + mi := &file_atelet_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1065,7 +1148,7 @@ func (x *ActorMetadataDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorMetadataDataSource.ProtoReflect.Descriptor instead. func (*ActorMetadataDataSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{14} + return file_atelet_proto_rawDescGZIP(), []int{16} } func (x *ActorMetadataDataSource) GetItems() []*ActorMetadataItem { @@ -1091,7 +1174,7 @@ type TrustBundleDataSource struct { func (x *TrustBundleDataSource) Reset() { *x = TrustBundleDataSource{} - mi := &file_atelet_proto_msgTypes[15] + mi := &file_atelet_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1103,7 +1186,7 @@ func (x *TrustBundleDataSource) String() string { func (*TrustBundleDataSource) ProtoMessage() {} func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[15] + mi := &file_atelet_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1116,7 +1199,7 @@ func (x *TrustBundleDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use TrustBundleDataSource.ProtoReflect.Descriptor instead. func (*TrustBundleDataSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{15} + return file_atelet_proto_rawDescGZIP(), []int{17} } func (x *TrustBundleDataSource) GetPath() string { @@ -1146,7 +1229,7 @@ type SystemInfoDataSource struct { func (x *SystemInfoDataSource) Reset() { *x = SystemInfoDataSource{} - mi := &file_atelet_proto_msgTypes[16] + mi := &file_atelet_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1158,7 +1241,7 @@ func (x *SystemInfoDataSource) String() string { func (*SystemInfoDataSource) ProtoMessage() {} func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[16] + mi := &file_atelet_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1171,7 +1254,7 @@ func (x *SystemInfoDataSource) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoDataSource.ProtoReflect.Descriptor instead. func (*SystemInfoDataSource) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{16} + return file_atelet_proto_rawDescGZIP(), []int{18} } func (x *SystemInfoDataSource) GetDataSource() isSystemInfoDataSource_DataSource { @@ -1227,7 +1310,7 @@ type SystemInfoVolume struct { func (x *SystemInfoVolume) Reset() { *x = SystemInfoVolume{} - mi := &file_atelet_proto_msgTypes[17] + mi := &file_atelet_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1239,7 +1322,7 @@ func (x *SystemInfoVolume) String() string { func (*SystemInfoVolume) ProtoMessage() {} func (x *SystemInfoVolume) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[17] + mi := &file_atelet_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1252,7 +1335,7 @@ func (x *SystemInfoVolume) ProtoReflect() protoreflect.Message { // Deprecated: Use SystemInfoVolume.ProtoReflect.Descriptor instead. func (*SystemInfoVolume) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{17} + return file_atelet_proto_rawDescGZIP(), []int{19} } func (x *SystemInfoVolume) GetDataSources() []*SystemInfoDataSource { @@ -1278,7 +1361,7 @@ type Volume struct { func (x *Volume) Reset() { *x = Volume{} - mi := &file_atelet_proto_msgTypes[18] + mi := &file_atelet_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1290,7 +1373,7 @@ func (x *Volume) String() string { func (*Volume) ProtoMessage() {} func (x *Volume) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[18] + mi := &file_atelet_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1303,7 +1386,7 @@ func (x *Volume) ProtoReflect() protoreflect.Message { // Deprecated: Use Volume.ProtoReflect.Descriptor instead. func (*Volume) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{18} + return file_atelet_proto_rawDescGZIP(), []int{20} } func (x *Volume) GetName() string { @@ -1394,7 +1477,7 @@ type VolumeMount struct { func (x *VolumeMount) Reset() { *x = VolumeMount{} - mi := &file_atelet_proto_msgTypes[19] + mi := &file_atelet_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1406,7 +1489,7 @@ func (x *VolumeMount) String() string { func (*VolumeMount) ProtoMessage() {} func (x *VolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[19] + mi := &file_atelet_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1419,7 +1502,7 @@ func (x *VolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. func (*VolumeMount) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{19} + return file_atelet_proto_rawDescGZIP(), []int{21} } func (x *VolumeMount) GetName() string { @@ -1455,7 +1538,7 @@ type Container struct { func (x *Container) Reset() { *x = Container{} - mi := &file_atelet_proto_msgTypes[20] + mi := &file_atelet_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1467,7 +1550,7 @@ func (x *Container) String() string { func (*Container) ProtoMessage() {} func (x *Container) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[20] + mi := &file_atelet_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1480,7 +1563,7 @@ func (x *Container) ProtoReflect() protoreflect.Message { // Deprecated: Use Container.ProtoReflect.Descriptor instead. func (*Container) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{20} + return file_atelet_proto_rawDescGZIP(), []int{22} } func (x *Container) GetName() string { @@ -1556,7 +1639,7 @@ type SecurityContext struct { func (x *SecurityContext) Reset() { *x = SecurityContext{} - mi := &file_atelet_proto_msgTypes[21] + mi := &file_atelet_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1568,7 +1651,7 @@ func (x *SecurityContext) String() string { func (*SecurityContext) ProtoMessage() {} func (x *SecurityContext) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[21] + mi := &file_atelet_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1581,7 +1664,7 @@ func (x *SecurityContext) ProtoReflect() protoreflect.Message { // Deprecated: Use SecurityContext.ProtoReflect.Descriptor instead. func (*SecurityContext) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{21} + return file_atelet_proto_rawDescGZIP(), []int{23} } func (x *SecurityContext) GetCapabilities() *Capabilities { @@ -1603,7 +1686,7 @@ type Capabilities struct { func (x *Capabilities) Reset() { *x = Capabilities{} - mi := &file_atelet_proto_msgTypes[22] + mi := &file_atelet_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1615,7 +1698,7 @@ func (x *Capabilities) String() string { func (*Capabilities) ProtoMessage() {} func (x *Capabilities) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[22] + mi := &file_atelet_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1628,7 +1711,7 @@ func (x *Capabilities) ProtoReflect() protoreflect.Message { // Deprecated: Use Capabilities.ProtoReflect.Descriptor instead. func (*Capabilities) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{22} + return file_atelet_proto_rawDescGZIP(), []int{24} } func (x *Capabilities) GetAdd() []string { @@ -1659,7 +1742,7 @@ type ResourceLimits struct { func (x *ResourceLimits) Reset() { *x = ResourceLimits{} - mi := &file_atelet_proto_msgTypes[23] + mi := &file_atelet_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1671,7 +1754,7 @@ func (x *ResourceLimits) String() string { func (*ResourceLimits) ProtoMessage() {} func (x *ResourceLimits) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[23] + mi := &file_atelet_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1684,7 +1767,7 @@ func (x *ResourceLimits) ProtoReflect() protoreflect.Message { // Deprecated: Use ResourceLimits.ProtoReflect.Descriptor instead. func (*ResourceLimits) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{23} + return file_atelet_proto_rawDescGZIP(), []int{25} } func (x *ResourceLimits) GetMemoryBytes() int64 { @@ -1711,7 +1794,7 @@ type EnvEntry struct { func (x *EnvEntry) Reset() { *x = EnvEntry{} - mi := &file_atelet_proto_msgTypes[24] + mi := &file_atelet_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1723,7 +1806,7 @@ func (x *EnvEntry) String() string { func (*EnvEntry) ProtoMessage() {} func (x *EnvEntry) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[24] + mi := &file_atelet_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1736,7 +1819,7 @@ func (x *EnvEntry) ProtoReflect() protoreflect.Message { // Deprecated: Use EnvEntry.ProtoReflect.Descriptor instead. func (*EnvEntry) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{24} + return file_atelet_proto_rawDescGZIP(), []int{26} } func (x *EnvEntry) GetName() string { @@ -1767,7 +1850,7 @@ type Readyz struct { func (x *Readyz) Reset() { *x = Readyz{} - mi := &file_atelet_proto_msgTypes[25] + mi := &file_atelet_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1779,7 +1862,7 @@ func (x *Readyz) String() string { func (*Readyz) ProtoMessage() {} func (x *Readyz) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[25] + mi := &file_atelet_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1792,7 +1875,7 @@ func (x *Readyz) ProtoReflect() protoreflect.Message { // Deprecated: Use Readyz.ProtoReflect.Descriptor instead. func (*Readyz) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{25} + return file_atelet_proto_rawDescGZIP(), []int{27} } func (x *Readyz) GetHttpGet() *HTTPGetAction { @@ -1822,7 +1905,7 @@ type HTTPGetAction struct { func (x *HTTPGetAction) Reset() { *x = HTTPGetAction{} - mi := &file_atelet_proto_msgTypes[26] + mi := &file_atelet_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1834,7 +1917,7 @@ func (x *HTTPGetAction) String() string { func (*HTTPGetAction) ProtoMessage() {} func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[26] + mi := &file_atelet_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1847,7 +1930,7 @@ func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPGetAction.ProtoReflect.Descriptor instead. func (*HTTPGetAction) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{26} + return file_atelet_proto_rawDescGZIP(), []int{28} } func (x *HTTPGetAction) GetPath() string { @@ -1872,7 +1955,7 @@ type RunResponse struct { func (x *RunResponse) Reset() { *x = RunResponse{} - mi := &file_atelet_proto_msgTypes[27] + mi := &file_atelet_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1884,7 +1967,7 @@ func (x *RunResponse) String() string { func (*RunResponse) ProtoMessage() {} func (x *RunResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[27] + mi := &file_atelet_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1897,7 +1980,7 @@ func (x *RunResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RunResponse.ProtoReflect.Descriptor instead. func (*RunResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{27} + return file_atelet_proto_rawDescGZIP(), []int{29} } type LocalCheckpointConfiguration struct { @@ -1913,7 +1996,7 @@ type LocalCheckpointConfiguration struct { func (x *LocalCheckpointConfiguration) Reset() { *x = LocalCheckpointConfiguration{} - mi := &file_atelet_proto_msgTypes[28] + mi := &file_atelet_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1925,7 +2008,7 @@ func (x *LocalCheckpointConfiguration) String() string { func (*LocalCheckpointConfiguration) ProtoMessage() {} func (x *LocalCheckpointConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[28] + mi := &file_atelet_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1938,7 +2021,7 @@ func (x *LocalCheckpointConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use LocalCheckpointConfiguration.ProtoReflect.Descriptor instead. func (*LocalCheckpointConfiguration) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{28} + return file_atelet_proto_rawDescGZIP(), []int{30} } func (x *LocalCheckpointConfiguration) GetSnapshotName() string { @@ -1959,7 +2042,7 @@ type ExternalCheckpointConfiguration struct { func (x *ExternalCheckpointConfiguration) Reset() { *x = ExternalCheckpointConfiguration{} - mi := &file_atelet_proto_msgTypes[29] + mi := &file_atelet_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1971,7 +2054,7 @@ func (x *ExternalCheckpointConfiguration) String() string { func (*ExternalCheckpointConfiguration) ProtoMessage() {} func (x *ExternalCheckpointConfiguration) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[29] + mi := &file_atelet_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1984,7 +2067,7 @@ func (x *ExternalCheckpointConfiguration) ProtoReflect() protoreflect.Message { // Deprecated: Use ExternalCheckpointConfiguration.ProtoReflect.Descriptor instead. func (*ExternalCheckpointConfiguration) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{29} + return file_atelet_proto_rawDescGZIP(), []int{31} } func (x *ExternalCheckpointConfiguration) GetSnapshotUri() string { @@ -2022,7 +2105,7 @@ type CheckpointRequest struct { func (x *CheckpointRequest) Reset() { *x = CheckpointRequest{} - mi := &file_atelet_proto_msgTypes[30] + mi := &file_atelet_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2034,7 +2117,7 @@ func (x *CheckpointRequest) String() string { func (*CheckpointRequest) ProtoMessage() {} func (x *CheckpointRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[30] + mi := &file_atelet_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2047,7 +2130,7 @@ func (x *CheckpointRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointRequest.ProtoReflect.Descriptor instead. func (*CheckpointRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{30} + return file_atelet_proto_rawDescGZIP(), []int{32} } func (x *CheckpointRequest) GetTargetAteomUid() string { @@ -2162,7 +2245,7 @@ type CheckpointResponse struct { func (x *CheckpointResponse) Reset() { *x = CheckpointResponse{} - mi := &file_atelet_proto_msgTypes[31] + mi := &file_atelet_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2174,7 +2257,7 @@ func (x *CheckpointResponse) String() string { func (*CheckpointResponse) ProtoMessage() {} func (x *CheckpointResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[31] + mi := &file_atelet_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2187,7 +2270,7 @@ func (x *CheckpointResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointResponse.ProtoReflect.Descriptor instead. func (*CheckpointResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{31} + return file_atelet_proto_rawDescGZIP(), []int{33} } type UploadPausedCheckpointRequest struct { @@ -2215,7 +2298,7 @@ type UploadPausedCheckpointRequest struct { func (x *UploadPausedCheckpointRequest) Reset() { *x = UploadPausedCheckpointRequest{} - mi := &file_atelet_proto_msgTypes[32] + mi := &file_atelet_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2227,7 +2310,7 @@ func (x *UploadPausedCheckpointRequest) String() string { func (*UploadPausedCheckpointRequest) ProtoMessage() {} func (x *UploadPausedCheckpointRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[32] + mi := &file_atelet_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2240,7 +2323,7 @@ func (x *UploadPausedCheckpointRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UploadPausedCheckpointRequest.ProtoReflect.Descriptor instead. func (*UploadPausedCheckpointRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{32} + return file_atelet_proto_rawDescGZIP(), []int{34} } func (x *UploadPausedCheckpointRequest) GetAtespace() string { @@ -2307,7 +2390,7 @@ type UploadPausedCheckpointResponse struct { func (x *UploadPausedCheckpointResponse) Reset() { *x = UploadPausedCheckpointResponse{} - mi := &file_atelet_proto_msgTypes[33] + mi := &file_atelet_proto_msgTypes[35] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2319,7 +2402,7 @@ func (x *UploadPausedCheckpointResponse) String() string { func (*UploadPausedCheckpointResponse) ProtoMessage() {} func (x *UploadPausedCheckpointResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[33] + mi := &file_atelet_proto_msgTypes[35] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2332,7 +2415,7 @@ func (x *UploadPausedCheckpointResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use UploadPausedCheckpointResponse.ProtoReflect.Descriptor instead. func (*UploadPausedCheckpointResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{33} + return file_atelet_proto_rawDescGZIP(), []int{35} } type RestoreRequest struct { @@ -2378,7 +2461,7 @@ type RestoreRequest struct { func (x *RestoreRequest) Reset() { *x = RestoreRequest{} - mi := &file_atelet_proto_msgTypes[34] + mi := &file_atelet_proto_msgTypes[36] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2390,7 +2473,7 @@ func (x *RestoreRequest) String() string { func (*RestoreRequest) ProtoMessage() {} func (x *RestoreRequest) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[34] + mi := &file_atelet_proto_msgTypes[36] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2403,7 +2486,7 @@ func (x *RestoreRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreRequest.ProtoReflect.Descriptor instead. func (*RestoreRequest) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{34} + return file_atelet_proto_rawDescGZIP(), []int{36} } func (x *RestoreRequest) GetTargetAteomUid() string { @@ -2546,7 +2629,7 @@ type RestoreResponse struct { func (x *RestoreResponse) Reset() { *x = RestoreResponse{} - mi := &file_atelet_proto_msgTypes[35] + mi := &file_atelet_proto_msgTypes[37] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2558,7 +2641,7 @@ func (x *RestoreResponse) String() string { func (*RestoreResponse) ProtoMessage() {} func (x *RestoreResponse) ProtoReflect() protoreflect.Message { - mi := &file_atelet_proto_msgTypes[35] + mi := &file_atelet_proto_msgTypes[37] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2571,14 +2654,17 @@ func (x *RestoreResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreResponse.ProtoReflect.Descriptor instead. func (*RestoreResponse) Descriptor() ([]byte, []int) { - return file_atelet_proto_rawDescGZIP(), []int{35} + return file_atelet_proto_rawDescGZIP(), []int{37} } var File_atelet_proto protoreflect.FileDescriptor const file_atelet_proto_rawDesc = "" + "\n" + - "\fatelet.proto\x12\x06atelet\"\x8b\x01\n" + + "\fatelet.proto\x12\x06atelet\x1a\x1fpkg/proto/ateapipb/ateapi.proto\"O\n" + + "\x18SetWorkerCapacityRequest\x123\n" + + "\bcapacity\x18\x01 \x01(\v2\x17.ateapi.WorkerResourcesR\bcapacity\"\x1b\n" + + "\x19SetWorkerCapacityResponse\"\x8b\x01\n" + "\x1bMintActorCertificateRequest\x12>\n" + "\x1bcertificate_signing_request\x18\x01 \x01(\fR\x19certificateSigningRequest\x12,\n" + "\x12expected_actor_uid\x18\x02 \x01(\tR\x10expectedActorUid\"M\n" + @@ -2769,7 +2855,9 @@ const file_atelet_proto_rawDesc = "" + "\x13SNAPSHOT_SCOPE_DATA\x10\x02\x12!\n" + "\x1dSNAPSHOT_SCOPE_DATA_ON_GOLDEN\x10\x032w\n" + "\x10CredentialBroker\x12c\n" + - "\x14MintActorCertificate\x12#.atelet.MintActorCertificateRequest\x1a$.atelet.MintActorCertificateResponse\"\x002\xf3\x02\n" + + "\x14MintActorCertificate\x12#.atelet.MintActorCertificateRequest\x1a$.atelet.MintActorCertificateResponse\"\x002l\n" + + "\x0eWorkerCapacity\x12Z\n" + + "\x11SetWorkerCapacity\x12 .atelet.SetWorkerCapacityRequest\x1a!.atelet.SetWorkerCapacityResponse\"\x002\xf3\x02\n" + "\vAteomHerder\x120\n" + "\x03Run\x12\x12.atelet.RunRequest\x1a\x13.atelet.RunResponse\"\x00\x12E\n" + "\n" + @@ -2791,108 +2879,114 @@ func file_atelet_proto_rawDescGZIP() []byte { } var file_atelet_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_atelet_proto_msgTypes = make([]protoimpl.MessageInfo, 39) +var file_atelet_proto_msgTypes = make([]protoimpl.MessageInfo, 41) var file_atelet_proto_goTypes = []any{ (ActorMetadataField)(0), // 0: atelet.ActorMetadataField (CheckpointType)(0), // 1: atelet.CheckpointType (SnapshotScope)(0), // 2: atelet.SnapshotScope - (*MintActorCertificateRequest)(nil), // 3: atelet.MintActorCertificateRequest - (*MintActorCertificateResponse)(nil), // 4: atelet.MintActorCertificateResponse - (*TerminateRequest)(nil), // 5: atelet.TerminateRequest - (*TerminateResponse)(nil), // 6: atelet.TerminateResponse - (*RunRequest)(nil), // 7: atelet.RunRequest - (*EgressGateway)(nil), // 8: atelet.EgressGateway - (*AssetFile)(nil), // 9: atelet.AssetFile - (*ArchAssets)(nil), // 10: atelet.ArchAssets - (*SandboxAssets)(nil), // 11: atelet.SandboxAssets - (*WorkloadSpec)(nil), // 12: atelet.WorkloadSpec - (*DurableDirVolume)(nil), // 13: atelet.DurableDirVolume - (*ExternalVolumeSource)(nil), // 14: atelet.ExternalVolumeSource - (*ImageVolumeSource)(nil), // 15: atelet.ImageVolumeSource - (*ActorMetadataItem)(nil), // 16: atelet.ActorMetadataItem - (*ActorMetadataDataSource)(nil), // 17: atelet.ActorMetadataDataSource - (*TrustBundleDataSource)(nil), // 18: atelet.TrustBundleDataSource - (*SystemInfoDataSource)(nil), // 19: atelet.SystemInfoDataSource - (*SystemInfoVolume)(nil), // 20: atelet.SystemInfoVolume - (*Volume)(nil), // 21: atelet.Volume - (*VolumeMount)(nil), // 22: atelet.VolumeMount - (*Container)(nil), // 23: atelet.Container - (*SecurityContext)(nil), // 24: atelet.SecurityContext - (*Capabilities)(nil), // 25: atelet.Capabilities - (*ResourceLimits)(nil), // 26: atelet.ResourceLimits - (*EnvEntry)(nil), // 27: atelet.EnvEntry - (*Readyz)(nil), // 28: atelet.Readyz - (*HTTPGetAction)(nil), // 29: atelet.HTTPGetAction - (*RunResponse)(nil), // 30: atelet.RunResponse - (*LocalCheckpointConfiguration)(nil), // 31: atelet.LocalCheckpointConfiguration - (*ExternalCheckpointConfiguration)(nil), // 32: atelet.ExternalCheckpointConfiguration - (*CheckpointRequest)(nil), // 33: atelet.CheckpointRequest - (*CheckpointResponse)(nil), // 34: atelet.CheckpointResponse - (*UploadPausedCheckpointRequest)(nil), // 35: atelet.UploadPausedCheckpointRequest - (*UploadPausedCheckpointResponse)(nil), // 36: atelet.UploadPausedCheckpointResponse - (*RestoreRequest)(nil), // 37: atelet.RestoreRequest - (*RestoreResponse)(nil), // 38: atelet.RestoreResponse - nil, // 39: atelet.ArchAssets.FilesEntry - nil, // 40: atelet.SandboxAssets.AssetsEntry - nil, // 41: atelet.ExternalVolumeSource.VolumeContextEntry + (*SetWorkerCapacityRequest)(nil), // 3: atelet.SetWorkerCapacityRequest + (*SetWorkerCapacityResponse)(nil), // 4: atelet.SetWorkerCapacityResponse + (*MintActorCertificateRequest)(nil), // 5: atelet.MintActorCertificateRequest + (*MintActorCertificateResponse)(nil), // 6: atelet.MintActorCertificateResponse + (*TerminateRequest)(nil), // 7: atelet.TerminateRequest + (*TerminateResponse)(nil), // 8: atelet.TerminateResponse + (*RunRequest)(nil), // 9: atelet.RunRequest + (*EgressGateway)(nil), // 10: atelet.EgressGateway + (*AssetFile)(nil), // 11: atelet.AssetFile + (*ArchAssets)(nil), // 12: atelet.ArchAssets + (*SandboxAssets)(nil), // 13: atelet.SandboxAssets + (*WorkloadSpec)(nil), // 14: atelet.WorkloadSpec + (*DurableDirVolume)(nil), // 15: atelet.DurableDirVolume + (*ExternalVolumeSource)(nil), // 16: atelet.ExternalVolumeSource + (*ImageVolumeSource)(nil), // 17: atelet.ImageVolumeSource + (*ActorMetadataItem)(nil), // 18: atelet.ActorMetadataItem + (*ActorMetadataDataSource)(nil), // 19: atelet.ActorMetadataDataSource + (*TrustBundleDataSource)(nil), // 20: atelet.TrustBundleDataSource + (*SystemInfoDataSource)(nil), // 21: atelet.SystemInfoDataSource + (*SystemInfoVolume)(nil), // 22: atelet.SystemInfoVolume + (*Volume)(nil), // 23: atelet.Volume + (*VolumeMount)(nil), // 24: atelet.VolumeMount + (*Container)(nil), // 25: atelet.Container + (*SecurityContext)(nil), // 26: atelet.SecurityContext + (*Capabilities)(nil), // 27: atelet.Capabilities + (*ResourceLimits)(nil), // 28: atelet.ResourceLimits + (*EnvEntry)(nil), // 29: atelet.EnvEntry + (*Readyz)(nil), // 30: atelet.Readyz + (*HTTPGetAction)(nil), // 31: atelet.HTTPGetAction + (*RunResponse)(nil), // 32: atelet.RunResponse + (*LocalCheckpointConfiguration)(nil), // 33: atelet.LocalCheckpointConfiguration + (*ExternalCheckpointConfiguration)(nil), // 34: atelet.ExternalCheckpointConfiguration + (*CheckpointRequest)(nil), // 35: atelet.CheckpointRequest + (*CheckpointResponse)(nil), // 36: atelet.CheckpointResponse + (*UploadPausedCheckpointRequest)(nil), // 37: atelet.UploadPausedCheckpointRequest + (*UploadPausedCheckpointResponse)(nil), // 38: atelet.UploadPausedCheckpointResponse + (*RestoreRequest)(nil), // 39: atelet.RestoreRequest + (*RestoreResponse)(nil), // 40: atelet.RestoreResponse + nil, // 41: atelet.ArchAssets.FilesEntry + nil, // 42: atelet.SandboxAssets.AssetsEntry + nil, // 43: atelet.ExternalVolumeSource.VolumeContextEntry + (*ateapipb.WorkerResources)(nil), // 44: ateapi.WorkerResources } var file_atelet_proto_depIdxs = []int32{ - 12, // 0: atelet.TerminateRequest.spec:type_name -> atelet.WorkloadSpec - 12, // 1: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec - 11, // 2: atelet.RunRequest.sandbox_assets:type_name -> atelet.SandboxAssets - 8, // 3: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway - 39, // 4: atelet.ArchAssets.files:type_name -> atelet.ArchAssets.FilesEntry - 40, // 5: atelet.SandboxAssets.assets:type_name -> atelet.SandboxAssets.AssetsEntry - 23, // 6: atelet.WorkloadSpec.containers:type_name -> atelet.Container - 21, // 7: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume - 41, // 8: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry - 0, // 9: atelet.ActorMetadataItem.field:type_name -> atelet.ActorMetadataField - 16, // 10: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem - 17, // 11: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource - 18, // 12: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource - 19, // 13: atelet.SystemInfoVolume.data_sources:type_name -> atelet.SystemInfoDataSource - 13, // 14: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolume - 14, // 15: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource - 20, // 16: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolume - 15, // 17: atelet.Volume.image:type_name -> atelet.ImageVolumeSource - 27, // 18: atelet.Container.env:type_name -> atelet.EnvEntry - 28, // 19: atelet.Container.readyz:type_name -> atelet.Readyz - 22, // 20: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount - 24, // 21: atelet.Container.security_context:type_name -> atelet.SecurityContext - 26, // 22: atelet.Container.resources:type_name -> atelet.ResourceLimits - 25, // 23: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities - 29, // 24: atelet.Readyz.http_get:type_name -> atelet.HTTPGetAction - 12, // 25: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec - 1, // 26: atelet.CheckpointRequest.type:type_name -> atelet.CheckpointType - 31, // 27: atelet.CheckpointRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration - 32, // 28: atelet.CheckpointRequest.external_config:type_name -> atelet.ExternalCheckpointConfiguration - 2, // 29: atelet.CheckpointRequest.scope:type_name -> atelet.SnapshotScope - 2, // 30: atelet.UploadPausedCheckpointRequest.desired_scope:type_name -> atelet.SnapshotScope - 12, // 31: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec - 1, // 32: atelet.RestoreRequest.type:type_name -> atelet.CheckpointType - 31, // 33: atelet.RestoreRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration - 32, // 34: atelet.RestoreRequest.external_config:type_name -> atelet.ExternalCheckpointConfiguration - 2, // 35: atelet.RestoreRequest.scope:type_name -> atelet.SnapshotScope - 8, // 36: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway - 9, // 37: atelet.ArchAssets.FilesEntry.value:type_name -> atelet.AssetFile - 10, // 38: atelet.SandboxAssets.AssetsEntry.value:type_name -> atelet.ArchAssets - 3, // 39: atelet.CredentialBroker.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest - 7, // 40: atelet.AteomHerder.Run:input_type -> atelet.RunRequest - 33, // 41: atelet.AteomHerder.Checkpoint:input_type -> atelet.CheckpointRequest - 37, // 42: atelet.AteomHerder.Restore:input_type -> atelet.RestoreRequest - 35, // 43: atelet.AteomHerder.UploadPausedCheckpoint:input_type -> atelet.UploadPausedCheckpointRequest - 5, // 44: atelet.AteomHerder.Terminate:input_type -> atelet.TerminateRequest - 4, // 45: atelet.CredentialBroker.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse - 30, // 46: atelet.AteomHerder.Run:output_type -> atelet.RunResponse - 34, // 47: atelet.AteomHerder.Checkpoint:output_type -> atelet.CheckpointResponse - 38, // 48: atelet.AteomHerder.Restore:output_type -> atelet.RestoreResponse - 36, // 49: atelet.AteomHerder.UploadPausedCheckpoint:output_type -> atelet.UploadPausedCheckpointResponse - 6, // 50: atelet.AteomHerder.Terminate:output_type -> atelet.TerminateResponse - 45, // [45:51] is the sub-list for method output_type - 39, // [39:45] is the sub-list for method input_type - 39, // [39:39] is the sub-list for extension type_name - 39, // [39:39] is the sub-list for extension extendee - 0, // [0:39] is the sub-list for field type_name + 44, // 0: atelet.SetWorkerCapacityRequest.capacity:type_name -> ateapi.WorkerResources + 14, // 1: atelet.TerminateRequest.spec:type_name -> atelet.WorkloadSpec + 14, // 2: atelet.RunRequest.spec:type_name -> atelet.WorkloadSpec + 13, // 3: atelet.RunRequest.sandbox_assets:type_name -> atelet.SandboxAssets + 10, // 4: atelet.RunRequest.egress_gateway:type_name -> atelet.EgressGateway + 41, // 5: atelet.ArchAssets.files:type_name -> atelet.ArchAssets.FilesEntry + 42, // 6: atelet.SandboxAssets.assets:type_name -> atelet.SandboxAssets.AssetsEntry + 25, // 7: atelet.WorkloadSpec.containers:type_name -> atelet.Container + 23, // 8: atelet.WorkloadSpec.volumes:type_name -> atelet.Volume + 43, // 9: atelet.ExternalVolumeSource.volume_context:type_name -> atelet.ExternalVolumeSource.VolumeContextEntry + 0, // 10: atelet.ActorMetadataItem.field:type_name -> atelet.ActorMetadataField + 18, // 11: atelet.ActorMetadataDataSource.items:type_name -> atelet.ActorMetadataItem + 19, // 12: atelet.SystemInfoDataSource.actor_metadata:type_name -> atelet.ActorMetadataDataSource + 20, // 13: atelet.SystemInfoDataSource.trust_bundle:type_name -> atelet.TrustBundleDataSource + 21, // 14: atelet.SystemInfoVolume.data_sources:type_name -> atelet.SystemInfoDataSource + 15, // 15: atelet.Volume.durable_dir:type_name -> atelet.DurableDirVolume + 16, // 16: atelet.Volume.external:type_name -> atelet.ExternalVolumeSource + 22, // 17: atelet.Volume.system_info:type_name -> atelet.SystemInfoVolume + 17, // 18: atelet.Volume.image:type_name -> atelet.ImageVolumeSource + 29, // 19: atelet.Container.env:type_name -> atelet.EnvEntry + 30, // 20: atelet.Container.readyz:type_name -> atelet.Readyz + 24, // 21: atelet.Container.volume_mounts:type_name -> atelet.VolumeMount + 26, // 22: atelet.Container.security_context:type_name -> atelet.SecurityContext + 28, // 23: atelet.Container.resources:type_name -> atelet.ResourceLimits + 27, // 24: atelet.SecurityContext.capabilities:type_name -> atelet.Capabilities + 31, // 25: atelet.Readyz.http_get:type_name -> atelet.HTTPGetAction + 14, // 26: atelet.CheckpointRequest.spec:type_name -> atelet.WorkloadSpec + 1, // 27: atelet.CheckpointRequest.type:type_name -> atelet.CheckpointType + 33, // 28: atelet.CheckpointRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration + 34, // 29: atelet.CheckpointRequest.external_config:type_name -> atelet.ExternalCheckpointConfiguration + 2, // 30: atelet.CheckpointRequest.scope:type_name -> atelet.SnapshotScope + 2, // 31: atelet.UploadPausedCheckpointRequest.desired_scope:type_name -> atelet.SnapshotScope + 14, // 32: atelet.RestoreRequest.spec:type_name -> atelet.WorkloadSpec + 1, // 33: atelet.RestoreRequest.type:type_name -> atelet.CheckpointType + 33, // 34: atelet.RestoreRequest.local_config:type_name -> atelet.LocalCheckpointConfiguration + 34, // 35: atelet.RestoreRequest.external_config:type_name -> atelet.ExternalCheckpointConfiguration + 2, // 36: atelet.RestoreRequest.scope:type_name -> atelet.SnapshotScope + 10, // 37: atelet.RestoreRequest.egress_gateway:type_name -> atelet.EgressGateway + 11, // 38: atelet.ArchAssets.FilesEntry.value:type_name -> atelet.AssetFile + 12, // 39: atelet.SandboxAssets.AssetsEntry.value:type_name -> atelet.ArchAssets + 5, // 40: atelet.CredentialBroker.MintActorCertificate:input_type -> atelet.MintActorCertificateRequest + 3, // 41: atelet.WorkerCapacity.SetWorkerCapacity:input_type -> atelet.SetWorkerCapacityRequest + 9, // 42: atelet.AteomHerder.Run:input_type -> atelet.RunRequest + 35, // 43: atelet.AteomHerder.Checkpoint:input_type -> atelet.CheckpointRequest + 39, // 44: atelet.AteomHerder.Restore:input_type -> atelet.RestoreRequest + 37, // 45: atelet.AteomHerder.UploadPausedCheckpoint:input_type -> atelet.UploadPausedCheckpointRequest + 7, // 46: atelet.AteomHerder.Terminate:input_type -> atelet.TerminateRequest + 6, // 47: atelet.CredentialBroker.MintActorCertificate:output_type -> atelet.MintActorCertificateResponse + 4, // 48: atelet.WorkerCapacity.SetWorkerCapacity:output_type -> atelet.SetWorkerCapacityResponse + 32, // 49: atelet.AteomHerder.Run:output_type -> atelet.RunResponse + 36, // 50: atelet.AteomHerder.Checkpoint:output_type -> atelet.CheckpointResponse + 40, // 51: atelet.AteomHerder.Restore:output_type -> atelet.RestoreResponse + 38, // 52: atelet.AteomHerder.UploadPausedCheckpoint:output_type -> atelet.UploadPausedCheckpointResponse + 8, // 53: atelet.AteomHerder.Terminate:output_type -> atelet.TerminateResponse + 47, // [47:54] is the sub-list for method output_type + 40, // [40:47] is the sub-list for method input_type + 40, // [40:40] is the sub-list for extension type_name + 40, // [40:40] is the sub-list for extension extendee + 0, // [0:40] is the sub-list for field type_name } func init() { file_atelet_proto_init() } @@ -2900,22 +2994,22 @@ func file_atelet_proto_init() { if File_atelet_proto != nil { return } - file_atelet_proto_msgTypes[4].OneofWrappers = []any{} - file_atelet_proto_msgTypes[16].OneofWrappers = []any{ + file_atelet_proto_msgTypes[6].OneofWrappers = []any{} + file_atelet_proto_msgTypes[18].OneofWrappers = []any{ (*SystemInfoDataSource_ActorMetadata)(nil), (*SystemInfoDataSource_TrustBundle)(nil), } - file_atelet_proto_msgTypes[18].OneofWrappers = []any{ + file_atelet_proto_msgTypes[20].OneofWrappers = []any{ (*Volume_DurableDir)(nil), (*Volume_External)(nil), (*Volume_SystemInfo)(nil), (*Volume_Image)(nil), } - file_atelet_proto_msgTypes[30].OneofWrappers = []any{ + file_atelet_proto_msgTypes[32].OneofWrappers = []any{ (*CheckpointRequest_LocalConfig)(nil), (*CheckpointRequest_ExternalConfig)(nil), } - file_atelet_proto_msgTypes[34].OneofWrappers = []any{ + file_atelet_proto_msgTypes[36].OneofWrappers = []any{ (*RestoreRequest_LocalConfig)(nil), (*RestoreRequest_ExternalConfig)(nil), } @@ -2925,9 +3019,9 @@ func file_atelet_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_atelet_proto_rawDesc), len(file_atelet_proto_rawDesc)), NumEnums: 3, - NumMessages: 39, + NumMessages: 41, NumExtensions: 0, - NumServices: 2, + NumServices: 3, }, GoTypes: file_atelet_proto_goTypes, DependencyIndexes: file_atelet_proto_depIdxs, diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index aa3fbbcf2a..869e83fccb 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -18,11 +18,30 @@ package atelet; option go_package = "github.com/agent-substrate/substrate/internal/proto/ateletpb"; +import "pkg/proto/ateapipb/ateapi.proto"; + // CredentialBroker gives an authenticated worker its current actor credential. service CredentialBroker { rpc MintActorCertificate(MintActorCertificateRequest) returns (MintActorCertificateResponse) {} } +// WorkerCapacity is how a worker tells the node-local atelet what it can +// supply to the actors it hosts, for atelet to forward to the control plane's +// WorkerService.SetWorkerCapacity, which this mirrors. The worker is identified +// by its mTLS certificate, never by the request. +service WorkerCapacity { + rpc SetWorkerCapacity(SetWorkerCapacityRequest) returns (SetWorkerCapacityResponse) {} +} + +message SetWorkerCapacityRequest { + // What the worker can supply, in the same vocabulary the control plane + // records and an ActorTemplate asks in. + ateapi.WorkerResources capacity = 1; +} + +message SetWorkerCapacityResponse { +} + message MintActorCertificateRequest { // DER-encoded PKCS #10 certificate signing request. Atunnel retains the // corresponding private key. diff --git a/internal/proto/ateletpb/atelet_grpc.pb.go b/internal/proto/ateletpb/atelet_grpc.pb.go index 4a84312a84..b90612c2b8 100644 --- a/internal/proto/ateletpb/atelet_grpc.pb.go +++ b/internal/proto/ateletpb/atelet_grpc.pb.go @@ -138,6 +138,118 @@ var CredentialBroker_ServiceDesc = grpc.ServiceDesc{ Metadata: "atelet.proto", } +const ( + WorkerCapacity_SetWorkerCapacity_FullMethodName = "/atelet.WorkerCapacity/SetWorkerCapacity" +) + +// WorkerCapacityClient is the client API for WorkerCapacity service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// WorkerCapacity is how a worker tells the node-local atelet what it can +// supply to the actors it hosts, for atelet to forward to the control plane's +// WorkerService.SetWorkerCapacity, which this mirrors. The worker is identified +// by its mTLS certificate, never by the request. +type WorkerCapacityClient interface { + SetWorkerCapacity(ctx context.Context, in *SetWorkerCapacityRequest, opts ...grpc.CallOption) (*SetWorkerCapacityResponse, error) +} + +type workerCapacityClient struct { + cc grpc.ClientConnInterface +} + +func NewWorkerCapacityClient(cc grpc.ClientConnInterface) WorkerCapacityClient { + return &workerCapacityClient{cc} +} + +func (c *workerCapacityClient) SetWorkerCapacity(ctx context.Context, in *SetWorkerCapacityRequest, opts ...grpc.CallOption) (*SetWorkerCapacityResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SetWorkerCapacityResponse) + err := c.cc.Invoke(ctx, WorkerCapacity_SetWorkerCapacity_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// WorkerCapacityServer is the server API for WorkerCapacity service. +// All implementations must embed UnimplementedWorkerCapacityServer +// for forward compatibility. +// +// WorkerCapacity is how a worker tells the node-local atelet what it can +// supply to the actors it hosts, for atelet to forward to the control plane's +// WorkerService.SetWorkerCapacity, which this mirrors. The worker is identified +// by its mTLS certificate, never by the request. +type WorkerCapacityServer interface { + SetWorkerCapacity(context.Context, *SetWorkerCapacityRequest) (*SetWorkerCapacityResponse, error) + mustEmbedUnimplementedWorkerCapacityServer() +} + +// UnimplementedWorkerCapacityServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedWorkerCapacityServer struct{} + +func (UnimplementedWorkerCapacityServer) SetWorkerCapacity(context.Context, *SetWorkerCapacityRequest) (*SetWorkerCapacityResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SetWorkerCapacity not implemented") +} +func (UnimplementedWorkerCapacityServer) mustEmbedUnimplementedWorkerCapacityServer() {} +func (UnimplementedWorkerCapacityServer) testEmbeddedByValue() {} + +// UnsafeWorkerCapacityServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to WorkerCapacityServer will +// result in compilation errors. +type UnsafeWorkerCapacityServer interface { + mustEmbedUnimplementedWorkerCapacityServer() +} + +func RegisterWorkerCapacityServer(s grpc.ServiceRegistrar, srv WorkerCapacityServer) { + // If the following call panics, it indicates UnimplementedWorkerCapacityServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&WorkerCapacity_ServiceDesc, srv) +} + +func _WorkerCapacity_SetWorkerCapacity_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetWorkerCapacityRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerCapacityServer).SetWorkerCapacity(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: WorkerCapacity_SetWorkerCapacity_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerCapacityServer).SetWorkerCapacity(ctx, req.(*SetWorkerCapacityRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// WorkerCapacity_ServiceDesc is the grpc.ServiceDesc for WorkerCapacity service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var WorkerCapacity_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "atelet.WorkerCapacity", + HandlerType: (*WorkerCapacityServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "SetWorkerCapacity", + Handler: _WorkerCapacity_SetWorkerCapacity_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "atelet.proto", +} + const ( AteomHerder_Run_FullMethodName = "/atelet.AteomHerder/Run" AteomHerder_Checkpoint_FullMethodName = "/atelet.AteomHerder/Checkpoint" diff --git a/internal/resources/quantities.go b/internal/resources/quantities.go new file mode 100644 index 0000000000..2e4072215f --- /dev/null +++ b/internal/resources/quantities.go @@ -0,0 +1,149 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "fmt" + "sort" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "k8s.io/apimachinery/pkg/api/resource" +) + +// Quantities is the parsed form of an ateapipb.Resources: what an Actor asks +// for, what a Worker supplies, or the sum of what a Worker's Actors hold. The +// same type serves all three so they subtract, which is the point of naming +// Worker capacity the way an ActorTemplate names its limits. +// +// An absent name is none of that resource. A Worker reports every dimension it +// has, so a name missing from its capacity is one it cannot supply at all. +type Quantities map[string]resource.Quantity + +// ParseQuantities reads the wire form. It errors on a quantity it cannot parse +// rather than skipping it, so a malformed limit cannot silently become +// unconstrained. +func ParseQuantities(r *ateapipb.Resources) (Quantities, error) { + if len(r.GetLimits()) == 0 { + return nil, nil + } + out := make(Quantities, len(r.GetLimits())) + for _, limit := range r.GetLimits() { + q, err := resource.ParseQuantity(limit.GetQuantity()) + if err != nil { + return nil, fmt.Errorf("resource %s has an invalid quantity %q: %w", limit.GetName(), limit.GetQuantity(), err) + } + if existing, ok := out[limit.GetName()]; ok { + existing.Add(q) + out[limit.GetName()] = existing + continue + } + out[limit.GetName()] = q + } + return out, nil +} + +// Proto is the wire form, sorted by name. Sorting is what lets proto.Equal +// decide whether a report or a recomputed total actually changed anything; +// unsorted, equal sets would compare unequal and churn the record. +// +// A dimension that has reached zero is dropped: it constrains nothing, and +// keeping it would make an emptied total compare unequal to an absent one. +func (q Quantities) Proto() *ateapipb.Resources { + if len(q) == 0 { + return nil + } + names := make([]string, 0, len(q)) + for name, quantity := range q { + if quantity.IsZero() { + continue + } + names = append(names, name) + } + if len(names) == 0 { + return nil + } + sort.Strings(names) + out := &ateapipb.Resources{Limits: make([]*ateapipb.Limits, 0, len(names))} + for _, name := range names { + quantity := q[name] + out.Limits = append(out.Limits, &ateapipb.Limits{Name: name, Quantity: quantity.String()}) + } + return out +} + +// ResourceCPU and ResourceMemory are the two dimensions everything declares +// today, named as Kubernetes names them so an ActorTemplate's limits and a +// Worker's capacity meet under the same keys. +const ( + ResourceCPU = "cpu" + ResourceMemory = "memory" +) + +// CPUMemory is the Resources for those two dimensions, in the units the +// runtimes deal in. A zero dimension is omitted, which reads as unconstrained. +func CPUMemory(cpuMilli, memoryBytes int64) *ateapipb.Resources { + q := Quantities{} + if cpuMilli != 0 { + q[ResourceCPU] = *resource.NewMilliQuantity(cpuMilli, resource.DecimalSI) + } + if memoryBytes != 0 { + q[ResourceMemory] = *resource.NewQuantity(memoryBytes, resource.BinarySI) + } + return q.Proto() +} + +// Add adds other into q, dimension by dimension. +func (q Quantities) Add(other Quantities) { + for name, quantity := range other { + existing, ok := q[name] + if !ok { + q[name] = quantity.DeepCopy() + continue + } + existing.Add(quantity) + q[name] = existing + } +} + +// Sub subtracts other from q, dimension by dimension. +func (q Quantities) Sub(other Quantities) { + for name, quantity := range other { + existing, ok := q[name] + if !ok { + neg := quantity.DeepCopy() + neg.Neg() + q[name] = neg + continue + } + existing.Sub(quantity) + q[name] = existing + } +} + +// Covers reports whether q leaves room for want in every dimension want names. +// +// A dimension q does not name is none of it, not any amount of it: a Worker +// reports everything it has, so silence about GPUs means it has no GPUs and +// cannot take an Actor asking for one. A dimension want does not name asks for +// nothing. +func (q Quantities) Covers(want Quantities) bool { + for name, need := range want { + have := q[name] // absent reads as the zero quantity + if have.Cmp(need) < 0 { + return false + } + } + return true +} diff --git a/internal/resources/quantities_test.go b/internal/resources/quantities_test.go new file mode 100644 index 0000000000..34aa1b2f24 --- /dev/null +++ b/internal/resources/quantities_test.go @@ -0,0 +1,177 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "testing" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/protobuf/proto" + "k8s.io/apimachinery/pkg/api/resource" +) + +func limits(pairs ...string) *ateapipb.Resources { + out := &ateapipb.Resources{} + for i := 0; i < len(pairs); i += 2 { + out.Limits = append(out.Limits, &ateapipb.Limits{Name: pairs[i], Quantity: pairs[i+1]}) + } + return out +} + +// The whole reason Proto sorts. Capacity is compared with proto.Equal to decide +// whether a report changed anything, and a repeated field compares positionally: +// unsorted, two equal sets would look different and write on every report. +func TestProtoIsOrderIndependent(t *testing.T) { + a, err := ParseQuantities(limits("cpu", "2", "memory", "8Gi")) + if err != nil { + t.Fatalf("ParseQuantities: %v", err) + } + b, err := ParseQuantities(limits("memory", "8Gi", "cpu", "2")) + if err != nil { + t.Fatalf("ParseQuantities: %v", err) + } + if !proto.Equal(a.Proto(), b.Proto()) { + t.Errorf("the same set in two orders round-trips unequal:\n%v\n%v", a.Proto(), b.Proto()) + } +} + +// A dimension that reaches zero is dropped, so a Worker whose Actors all left +// compares equal to one that never held any. +func TestProtoDropsEmptyDimensions(t *testing.T) { + q := Quantities{"cpu": resource.MustParse("0")} + if got := q.Proto(); got != nil { + t.Errorf("Proto() = %v for an all-zero set, want nil", got) + } + if got := (Quantities{}).Proto(); got != nil { + t.Errorf("Proto() = %v for an empty set, want nil", got) + } +} + +func TestParseQuantities(t *testing.T) { + t.Run("an unparseable quantity is an error, not a skip", func(t *testing.T) { + if _, err := ParseQuantities(limits("cpu", "banana")); err == nil { + t.Error("ParseQuantities() = nil error, want one") + } + }) + + t.Run("nothing declared parses to nothing", func(t *testing.T) { + got, err := ParseQuantities(nil) + if err != nil || got != nil { + t.Errorf("ParseQuantities(nil) = %v, %v, want nil, nil", got, err) + } + }) + + t.Run("a repeated name sums rather than shadowing", func(t *testing.T) { + got, err := ParseQuantities(limits("cpu", "1", "cpu", "500m")) + if err != nil { + t.Fatalf("ParseQuantities: %v", err) + } + cpu := got["cpu"] + if want := resource.MustParse("1500m"); cpu.Cmp(want) != 0 { + t.Errorf("cpu = %v, want %v", cpu, want) + } + }) +} + +func TestQuantitiesAddAndSub(t *testing.T) { + q := Quantities{} + q.Add(Quantities{"cpu": resource.MustParse("2"), "memory": resource.MustParse("4Gi")}) + q.Add(Quantities{"cpu": resource.MustParse("1")}) + cpu := q["cpu"] + if want := resource.MustParse("3"); cpu.Cmp(want) != 0 { + t.Errorf("cpu after adds = %v, want %v", cpu, want) + } + + q.Sub(Quantities{"cpu": resource.MustParse("3"), "memory": resource.MustParse("4Gi")}) + if got := q.Proto(); got != nil { + t.Errorf("Proto() = %v after subtracting everything back out, want nil", got) + } +} + +func TestQuantitiesCovers(t *testing.T) { + tests := []struct { + name string + have Quantities + want Quantities + ok bool + }{ + { + name: "enough of every dimension", + have: Quantities{"cpu": resource.MustParse("4"), "memory": resource.MustParse("8Gi")}, + want: Quantities{"cpu": resource.MustParse("2"), "memory": resource.MustParse("4Gi")}, + ok: true, + }, + { + name: "exactly enough", + have: Quantities{"cpu": resource.MustParse("2")}, + want: Quantities{"cpu": resource.MustParse("2")}, + ok: true, + }, + { + name: "short in one dimension", + have: Quantities{"cpu": resource.MustParse("4"), "memory": resource.MustParse("1Gi")}, + want: Quantities{"cpu": resource.MustParse("2"), "memory": resource.MustParse("4Gi")}, + ok: false, + }, + { + // A Worker reports everything it has, so a dimension it never + // reported is one it has none of. Asking for a GPU must not land on + // a Worker that never said it had one. + name: "a dimension the worker never reported is none of it", + have: Quantities{"cpu": resource.MustParse("4")}, + want: Quantities{"cpu": resource.MustParse("2"), "nvidia.com/gpu": resource.MustParse("1")}, + ok: false, + }, + { + // Asking for zero of something absent is still satisfiable. + name: "asking for none of an absent dimension fits", + have: Quantities{"cpu": resource.MustParse("4")}, + want: Quantities{"nvidia.com/gpu": resource.MustParse("0")}, + ok: true, + }, + { + name: "an actor asking for nothing fits anywhere", + have: Quantities{"cpu": resource.MustParse("0")}, + want: nil, + ok: true, + }, + { + // What is left goes negative once a Worker is overcommitted, and + // nothing more may be placed on it. + name: "an overcommitted dimension covers nothing", + have: Quantities{"cpu": *resource.NewMilliQuantity(-500, resource.DecimalSI)}, + want: Quantities{"cpu": resource.MustParse("1")}, + ok: false, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := tc.have.Covers(tc.want); got != tc.ok { + t.Errorf("Covers() = %v, want %v", got, tc.ok) + } + }) + } +} + +func TestCPUMemory(t *testing.T) { + got := CPUMemory(2500, 8<<30) + want := limits("cpu", "2500m", "memory", "8Gi") + if !proto.Equal(got, want) { + t.Errorf("CPUMemory() = %v, want %v", got, want) + } + if got := CPUMemory(0, 0); got != nil { + t.Errorf("CPUMemory(0, 0) = %v, want nil", got) + } +} diff --git a/internal/resources/worker.go b/internal/resources/worker.go new file mode 100644 index 0000000000..66a8131414 --- /dev/null +++ b/internal/resources/worker.go @@ -0,0 +1,80 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "fmt" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// AddToAllocated adjusts allocation by an assignment; sign is 1 or -1. +// It returns nil when allocation reaches zero. +func AddToAllocated(total *ateapipb.WorkerResources, assignment *ateapipb.ActorAssignment, sign int64) (*ateapipb.WorkerResources, error) { + held, err := ParseQuantities(total.GetResources()) + if err != nil { + return nil, fmt.Errorf("allocated: %w", err) + } + booked, err := ParseQuantities(assignment.GetResources()) + if err != nil { + return nil, fmt.Errorf("assignment for actor %s: %w", assignment.GetActorUid(), err) + } + if held == nil { + held = Quantities{} + } + if sign > 0 { + held.Add(booked) + } else { + held.Sub(booked) + } + + actors := total.GetActors() + int32(sign) + resources := held.Proto() + if actors == 0 && resources == nil { + return nil, nil + } + return &ateapipb.WorkerResources{Actors: actors, Resources: resources}, nil +} + +// SumAllocated is what a set of assignments takes from a Worker, or nil for +// none. Rebuilds the total rather than adjusting it, which is what the checks +// holding AddToAllocated to the assignments it counts compare against. +func SumAllocated(assignments []*ateapipb.ActorAssignment) (*ateapipb.WorkerResources, error) { + if len(assignments) == 0 { + return nil, nil + } + total := Quantities{} + for _, assignment := range assignments { + booked, err := ParseQuantities(assignment.GetResources()) + if err != nil { + return nil, fmt.Errorf("assignment for actor %s: %w", assignment.GetActorUid(), err) + } + total.Add(booked) + } + return &ateapipb.WorkerResources{Actors: int32(len(assignments)), Resources: total.Proto()}, nil +} + +// Allocation returns a Worker's allocation, creating the status and allocation +// it hangs from when they are absent. A Worker that has never been placed on +// nor reported carries neither. +func Allocation(worker *ateapipb.Worker) *ateapipb.WorkerAllocation { + if worker.Status == nil { + worker.Status = &ateapipb.WorkerStatus{} + } + if worker.Status.Allocation == nil { + worker.Status.Allocation = &ateapipb.WorkerAllocation{} + } + return worker.Status.Allocation +} diff --git a/internal/resources/worker_test.go b/internal/resources/worker_test.go new file mode 100644 index 0000000000..68590df846 --- /dev/null +++ b/internal/resources/worker_test.go @@ -0,0 +1,108 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resources + +import ( + "testing" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/protobuf/testing/protocmp" + + "github.com/google/go-cmp/cmp" +) + +func assignment(uid string, cpu, mem int64) *ateapipb.ActorAssignment { + return &ateapipb.ActorAssignment{ + Actor: &ateapipb.ObjectRef{Atespace: "demo", Name: uid}, + ActorUid: uid, + Resources: CPUMemory(cpu, mem), + } +} + +// mustAdd is AddToAllocated where the fixtures are known to parse. +func mustAdd(t *testing.T, total *ateapipb.WorkerResources, a *ateapipb.ActorAssignment, sign int64) *ateapipb.WorkerResources { + t.Helper() + got, err := AddToAllocated(total, a, sign) + if err != nil { + t.Fatalf("AddToAllocated(%v, %d): %v", a, sign, err) + } + return got +} + +// The total moves by one assignment's worth in each direction, and a Worker +// back to holding nothing carries no allocation at all rather than a zeroed +// message: emptied and never-filled have to be the same record. +func TestAddToAllocated(t *testing.T) { + var total *ateapipb.WorkerResources + total = mustAdd(t, total, assignment("a", 1000, 1<<30), +1) + total = mustAdd(t, total, assignment("b", 500, 2<<30), +1) + + want := &ateapipb.WorkerResources{Actors: 2, Resources: CPUMemory(1500, 3<<30)} + if diff := cmp.Diff(want, total, protocmp.Transform()); diff != "" { + t.Errorf("allocated mismatch (-want +got):\n%s", diff) + } + + total = mustAdd(t, total, assignment("b", 500, 2<<30), -1) + want = &ateapipb.WorkerResources{Actors: 1, Resources: CPUMemory(1000, 1<<30)} + if diff := cmp.Diff(want, total, protocmp.Transform()); diff != "" { + t.Errorf("allocated after release mismatch (-want +got):\n%s", diff) + } + + if total = mustAdd(t, total, assignment("a", 1000, 1<<30), -1); total != nil { + t.Errorf("allocated = %v for a Worker holding nothing, want nil", total) + } +} + +// An Actor that declared no limits reserves nothing but still costs a slot. +func TestAddToAllocatedCountsAnActorWithoutResources(t *testing.T) { + total := mustAdd(t, nil, &ateapipb.ActorAssignment{ActorUid: "a"}, +1) + want := &ateapipb.WorkerResources{Actors: 1} + if diff := cmp.Diff(want, total, protocmp.Transform()); diff != "" { + t.Errorf("allocated mismatch (-want +got):\n%s", diff) + } +} + +// A quantity that will not parse is refused rather than counted as nothing: +// silently booking zero would overcommit the Worker. +func TestAddToAllocatedRejectsAnUnparseableQuantity(t *testing.T) { + bad := &ateapipb.ActorAssignment{ + ActorUid: "a", + Resources: &ateapipb.Resources{Limits: []*ateapipb.Limits{{Name: "cpu", Quantity: "two"}}}, + } + if _, err := AddToAllocated(nil, bad, +1); err == nil { + t.Fatal("AddToAllocated() = nil error for an unparseable quantity, want one") + } +} + +// SumAllocated rebuilds what AddToAllocated adjusts; the contract tests hold the +// two to each other, so they have to agree on the empty case as well. +func TestSumAllocated(t *testing.T) { + got, err := SumAllocated(nil) + if err != nil { + t.Fatalf("SumAllocated(nil): %v", err) + } + if got != nil { + t.Errorf("SumAllocated(nil) = %v, want nil", got) + } + + got, err = SumAllocated([]*ateapipb.ActorAssignment{assignment("a", 1000, 1<<30), assignment("b", 500, 0)}) + if err != nil { + t.Fatalf("SumAllocated(): %v", err) + } + want := &ateapipb.WorkerResources{Actors: 2, Resources: CPUMemory(1500, 1<<30)} + if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { + t.Errorf("SumAllocated() mismatch (-want +got):\n%s", diff) + } +} diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index ea5121e4c9..4dc3e013ff 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -5382,6 +5382,137 @@ func (x *DeleteOptions) GetUid() string { return "" } +// ListWorkerActorAssignmentsRequest asks for a page of the Actors one Worker hosts. +type ListWorkerActorAssignmentsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The Worker whose Actors to list. atespace is always empty; Workers are + // global-scoped. + // + // +k8s:required + // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix + Worker *ObjectRef `protobuf:"bytes,1,opt,name=worker,proto3" json:"worker,omitempty"` + // Requested page size; the server may return fewer. If unspecified, defaults + // to a server-chosen value; values above 1000 are coerced to 1000. + // + // +k8s:optional + // +k8s:minimum=1 + PageSize int32 `protobuf:"varint,2,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` + // Pagination token from a previous ListWorkerActorAssignments response. + // Omit or leave empty for the first request. + // + // +k8s:optional + // +k8s:maxLength=256 + PageToken string `protobuf:"bytes,3,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListWorkerActorAssignmentsRequest) Reset() { + *x = ListWorkerActorAssignmentsRequest{} + mi := &file_ateapi_proto_msgTypes[75] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListWorkerActorAssignmentsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListWorkerActorAssignmentsRequest) ProtoMessage() {} + +func (x *ListWorkerActorAssignmentsRequest) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[75] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListWorkerActorAssignmentsRequest.ProtoReflect.Descriptor instead. +func (*ListWorkerActorAssignmentsRequest) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{75} +} + +func (x *ListWorkerActorAssignmentsRequest) GetWorker() *ObjectRef { + if x != nil { + return x.Worker + } + return nil +} + +func (x *ListWorkerActorAssignmentsRequest) GetPageSize() int32 { + if x != nil { + return x.PageSize + } + return 0 +} + +func (x *ListWorkerActorAssignmentsRequest) GetPageToken() string { + if x != nil { + return x.PageToken + } + return "" +} + +// ListWorkerActorAssignmentsResponse is one page of a Worker's Actors. +type ListWorkerActorAssignmentsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The Actors this page of the listing covers. + ActorAssignments []*ActorAssignment `protobuf:"bytes,1,rep,name=actor_assignments,json=actorAssignments,proto3" json:"actor_assignments,omitempty"` + // Pagination token for the next page. Empty if this is the last page. + NextPageToken string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListWorkerActorAssignmentsResponse) Reset() { + *x = ListWorkerActorAssignmentsResponse{} + mi := &file_ateapi_proto_msgTypes[76] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListWorkerActorAssignmentsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListWorkerActorAssignmentsResponse) ProtoMessage() {} + +func (x *ListWorkerActorAssignmentsResponse) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[76] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListWorkerActorAssignmentsResponse.ProtoReflect.Descriptor instead. +func (*ListWorkerActorAssignmentsResponse) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{76} +} + +func (x *ListWorkerActorAssignmentsResponse) GetActorAssignments() []*ActorAssignment { + if x != nil { + return x.ActorAssignments + } + return nil +} + +func (x *ListWorkerActorAssignmentsResponse) GetNextPageToken() string { + if x != nil { + return x.NextPageToken + } + return "" +} + type ListWorkersRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // Requested page size; the server may return fewer, or occasionally @@ -5403,7 +5534,7 @@ type ListWorkersRequest struct { func (x *ListWorkersRequest) Reset() { *x = ListWorkersRequest{} - mi := &file_ateapi_proto_msgTypes[75] + mi := &file_ateapi_proto_msgTypes[77] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5415,7 +5546,7 @@ func (x *ListWorkersRequest) String() string { func (*ListWorkersRequest) ProtoMessage() {} func (x *ListWorkersRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[75] + mi := &file_ateapi_proto_msgTypes[77] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5428,7 +5559,7 @@ func (x *ListWorkersRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkersRequest.ProtoReflect.Descriptor instead. func (*ListWorkersRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{75} + return file_ateapi_proto_rawDescGZIP(), []int{77} } func (x *ListWorkersRequest) GetPageSize() int32 { @@ -5457,7 +5588,7 @@ type ListWorkersResponse struct { func (x *ListWorkersResponse) Reset() { *x = ListWorkersResponse{} - mi := &file_ateapi_proto_msgTypes[76] + mi := &file_ateapi_proto_msgTypes[78] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5469,7 +5600,7 @@ func (x *ListWorkersResponse) String() string { func (*ListWorkersResponse) ProtoMessage() {} func (x *ListWorkersResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[76] + mi := &file_ateapi_proto_msgTypes[78] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5482,7 +5613,7 @@ func (x *ListWorkersResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkersResponse.ProtoReflect.Descriptor instead. func (*ListWorkersResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{76} + return file_ateapi_proto_rawDescGZIP(), []int{78} } func (x *ListWorkersResponse) GetWorkers() []*Worker { @@ -5512,7 +5643,7 @@ type GetWorkerRequest struct { func (x *GetWorkerRequest) Reset() { *x = GetWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[77] + mi := &file_ateapi_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5524,7 +5655,7 @@ func (x *GetWorkerRequest) String() string { func (*GetWorkerRequest) ProtoMessage() {} func (x *GetWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[77] + mi := &file_ateapi_proto_msgTypes[79] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5537,7 +5668,7 @@ func (x *GetWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkerRequest.ProtoReflect.Descriptor instead. func (*GetWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{77} + return file_ateapi_proto_rawDescGZIP(), []int{79} } func (x *GetWorkerRequest) GetWorker() *ObjectRef { @@ -5559,7 +5690,7 @@ type CreateWorkerRequest struct { func (x *CreateWorkerRequest) Reset() { *x = CreateWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[78] + mi := &file_ateapi_proto_msgTypes[80] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5571,7 +5702,7 @@ func (x *CreateWorkerRequest) String() string { func (*CreateWorkerRequest) ProtoMessage() {} func (x *CreateWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[78] + mi := &file_ateapi_proto_msgTypes[80] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5584,7 +5715,7 @@ func (x *CreateWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateWorkerRequest.ProtoReflect.Descriptor instead. func (*CreateWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{78} + return file_ateapi_proto_rawDescGZIP(), []int{80} } func (x *CreateWorkerRequest) GetWorker() *Worker { @@ -5619,7 +5750,7 @@ type UpdateWorkerRequest struct { func (x *UpdateWorkerRequest) Reset() { *x = UpdateWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[79] + mi := &file_ateapi_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5631,7 +5762,7 @@ func (x *UpdateWorkerRequest) String() string { func (*UpdateWorkerRequest) ProtoMessage() {} func (x *UpdateWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[79] + mi := &file_ateapi_proto_msgTypes[81] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5644,7 +5775,7 @@ func (x *UpdateWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateWorkerRequest.ProtoReflect.Descriptor instead. func (*UpdateWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{79} + return file_ateapi_proto_rawDescGZIP(), []int{81} } func (x *UpdateWorkerRequest) GetWorker() *Worker { @@ -5671,7 +5802,7 @@ type DeleteWorkerRequest struct { func (x *DeleteWorkerRequest) Reset() { *x = DeleteWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[80] + mi := &file_ateapi_proto_msgTypes[82] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5683,7 +5814,7 @@ func (x *DeleteWorkerRequest) String() string { func (*DeleteWorkerRequest) ProtoMessage() {} func (x *DeleteWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[80] + mi := &file_ateapi_proto_msgTypes[82] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5696,7 +5827,7 @@ func (x *DeleteWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteWorkerRequest.ProtoReflect.Descriptor instead. func (*DeleteWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{80} + return file_ateapi_proto_rawDescGZIP(), []int{82} } func (x *DeleteWorkerRequest) GetWorker() *ObjectRef { @@ -5726,7 +5857,7 @@ type DrainWorkerRequest struct { func (x *DrainWorkerRequest) Reset() { *x = DrainWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[81] + mi := &file_ateapi_proto_msgTypes[83] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5738,7 +5869,7 @@ func (x *DrainWorkerRequest) String() string { func (*DrainWorkerRequest) ProtoMessage() {} func (x *DrainWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[81] + mi := &file_ateapi_proto_msgTypes[83] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5751,7 +5882,7 @@ func (x *DrainWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainWorkerRequest.ProtoReflect.Descriptor instead. func (*DrainWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{81} + return file_ateapi_proto_rawDescGZIP(), []int{83} } func (x *DrainWorkerRequest) GetWorker() *ObjectRef { @@ -5789,7 +5920,7 @@ type ListActorsRequest struct { func (x *ListActorsRequest) Reset() { *x = ListActorsRequest{} - mi := &file_ateapi_proto_msgTypes[82] + mi := &file_ateapi_proto_msgTypes[84] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5801,7 +5932,7 @@ func (x *ListActorsRequest) String() string { func (*ListActorsRequest) ProtoMessage() {} func (x *ListActorsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[82] + mi := &file_ateapi_proto_msgTypes[84] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5814,7 +5945,7 @@ func (x *ListActorsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorsRequest.ProtoReflect.Descriptor instead. func (*ListActorsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{82} + return file_ateapi_proto_rawDescGZIP(), []int{84} } func (x *ListActorsRequest) GetAtespace() string { @@ -5850,7 +5981,7 @@ type ListActorsResponse struct { func (x *ListActorsResponse) Reset() { *x = ListActorsResponse{} - mi := &file_ateapi_proto_msgTypes[83] + mi := &file_ateapi_proto_msgTypes[85] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5862,7 +5993,7 @@ func (x *ListActorsResponse) String() string { func (*ListActorsResponse) ProtoMessage() {} func (x *ListActorsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[83] + mi := &file_ateapi_proto_msgTypes[85] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5875,7 +6006,7 @@ func (x *ListActorsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorsResponse.ProtoReflect.Descriptor instead. func (*ListActorsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{83} + return file_ateapi_proto_rawDescGZIP(), []int{85} } func (x *ListActorsResponse) GetActors() []*Actor { @@ -5952,17 +6083,9 @@ type Worker struct { // +k8s:eachKey=+k8s:format=k8s-label-key // +k8s:eachVal=+k8s:format=k8s-label-value Labels map[string]string `protobuf:"bytes,9,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - // The compute capacity this worker can give an actor sandbox. Immutable, set - // at creation: a worker pod's limits are fixed for its lifetime. An update - // replaces the worker rather than patching it, so a request that omits - // capacity is asking to clear it, which the immutability rule rejects. - // - // +k8s:optional - // +k8s:immutable - Capacity *WorkerCapacity `protobuf:"bytes,10,opt,name=capacity,proto3" json:"capacity,omitempty"` // Output-only server-managed state. Absent from Create/Update request // payloads; whatever a request carries here is ignored. DrainWorker is the - // only way a client moves state, and assignment is the scheduler's. + // only way a client moves state, and the assignments are the scheduler's. // // +k8s:optional Status *WorkerStatus `protobuf:"bytes,11,opt,name=status,proto3" json:"status,omitempty"` @@ -5972,7 +6095,7 @@ type Worker struct { func (x *Worker) Reset() { *x = Worker{} - mi := &file_ateapi_proto_msgTypes[84] + mi := &file_ateapi_proto_msgTypes[86] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5984,7 +6107,7 @@ func (x *Worker) String() string { func (*Worker) ProtoMessage() {} func (x *Worker) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[84] + mi := &file_ateapi_proto_msgTypes[86] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5997,7 +6120,7 @@ func (x *Worker) ProtoReflect() protoreflect.Message { // Deprecated: Use Worker.ProtoReflect.Descriptor instead. func (*Worker) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{84} + return file_ateapi_proto_rawDescGZIP(), []int{86} } func (x *Worker) GetMetadata() *ResourceMetadata { @@ -6063,13 +6186,6 @@ func (x *Worker) GetLabels() map[string]string { return nil } -func (x *Worker) GetCapacity() *WorkerCapacity { - if x != nil { - return x.Capacity - } - return nil -} - func (x *Worker) GetStatus() *WorkerStatus { if x != nil { return x.Status @@ -6083,17 +6199,18 @@ type WorkerStatus struct { // +k8s:minimum=1 // +k8s:maximum=2 # keep this in sync with the WorkerState enum State WorkerState `protobuf:"varint,1,opt,name=state,proto3,enum=ateapi.WorkerState" json:"state,omitempty"` - // The Actor currently bound to this Worker, if any. + // What this Worker has and what is spoken for. Observed, not requested: no + // client sets it, which is why it is status. // // +k8s:optional - Assignment *ActorAssignment `protobuf:"bytes,2,opt,name=assignment,proto3" json:"assignment,omitempty"` + Allocation *WorkerAllocation `protobuf:"bytes,2,opt,name=allocation,proto3" json:"allocation,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *WorkerStatus) Reset() { *x = WorkerStatus{} - mi := &file_ateapi_proto_msgTypes[85] + mi := &file_ateapi_proto_msgTypes[87] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6105,7 +6222,7 @@ func (x *WorkerStatus) String() string { func (*WorkerStatus) ProtoMessage() {} func (x *WorkerStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[85] + mi := &file_ateapi_proto_msgTypes[87] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6118,7 +6235,7 @@ func (x *WorkerStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerStatus.ProtoReflect.Descriptor instead. func (*WorkerStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{85} + return file_ateapi_proto_rawDescGZIP(), []int{87} } func (x *WorkerStatus) GetState() WorkerState { @@ -6128,50 +6245,121 @@ func (x *WorkerStatus) GetState() WorkerState { return WorkerState_WORKER_STATE_UNSPECIFIED } -func (x *WorkerStatus) GetAssignment() *ActorAssignment { +func (x *WorkerStatus) GetAllocation() *WorkerAllocation { + if x != nil { + return x.Allocation + } + return nil +} + +// WorkerAllocation is a Worker's total against its committed share, in one +// message because placement only ever reads the two together. +type WorkerAllocation struct { + state protoimpl.MessageState `protogen:"open.v1"` + // What the Worker can supply, as it reports through + // WorkerService.SetWorkerCapacity. + // + // Complete, not partial: a Worker reports every dimension it has, so a name + // missing here is one it cannot supply at all and no Actor asking for that + // name is placed on it. Shrinking below allocated stops new placements and + // evicts nothing. + // + // +k8s:optional + Capacity *WorkerResources `protobuf:"bytes,1,opt,name=capacity,proto3" json:"capacity,omitempty"` + // What the assignments consume of capacity. A running total because placement + // reads it for every Worker on every decision, and summing the assignments + // would cost the fleet's actor count each time. + // + // +k8s:optional + Allocated *WorkerResources `protobuf:"bytes,2,opt,name=allocated,proto3" json:"allocated,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WorkerAllocation) Reset() { + *x = WorkerAllocation{} + mi := &file_ateapi_proto_msgTypes[88] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WorkerAllocation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkerAllocation) ProtoMessage() {} + +func (x *WorkerAllocation) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[88] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkerAllocation.ProtoReflect.Descriptor instead. +func (*WorkerAllocation) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{88} +} + +func (x *WorkerAllocation) GetCapacity() *WorkerResources { if x != nil { - return x.Assignment + return x.Capacity } return nil } -// WorkerCapacity is the worker pod's compute capacity available to host an -// actor sandbox, taken from the ateom container's resource limits. The -// scheduler only places an actor on a worker whose capacity is >= the actor's -// declared resource limits. An unset message, or a zero field within it, means -// "unknown/unset" for that dimension: treated as unconstrained so placement is -// not blocked (matching the pre-capacity behavior). -type WorkerCapacity struct { +func (x *WorkerAllocation) GetAllocated() *WorkerResources { + if x != nil { + return x.Allocated + } + return nil +} + +// WorkerResources represents schedulable resources. +type WorkerResources struct { state protoimpl.MessageState `protogen:"open.v1"` - // CPU capacity in millicores (1000 = one core). + // Resources match the actor's definition of resources. + // They are matched by name, used for placement. + // + // Sorted by name, so equal capacities compare equal. // // +k8s:optional - // +k8s:minimum=1 - CpuMilli int64 `protobuf:"varint,1,opt,name=cpu_milli,json=cpuMilli,proto3" json:"cpu_milli,omitempty"` - // Memory capacity in bytes. + Resources *Resources `protobuf:"bytes,1,opt,name=resources,proto3" json:"resources,omitempty"` + // How many Actors may be bound at once. Not a resource because nothing + // divides it: every Actor costs exactly one. It bounds what one more Actor + // costs where CPU and memory do not — netns, mounts, file descriptors, blast + // radius. Kubernetes bounds a node the same way, with allocatable pods. + // + // Absent until the Worker reports, and a Worker that has reported no ceiling + // holds no Actors, like any other dimension. // // +k8s:optional // +k8s:minimum=1 - MemoryBytes int64 `protobuf:"varint,2,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` + Actors int32 `protobuf:"varint,2,opt,name=actors,proto3" json:"actors,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *WorkerCapacity) Reset() { - *x = WorkerCapacity{} - mi := &file_ateapi_proto_msgTypes[86] +func (x *WorkerResources) Reset() { + *x = WorkerResources{} + mi := &file_ateapi_proto_msgTypes[89] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *WorkerCapacity) String() string { +func (x *WorkerResources) String() string { return protoimpl.X.MessageStringOf(x) } -func (*WorkerCapacity) ProtoMessage() {} +func (*WorkerResources) ProtoMessage() {} -func (x *WorkerCapacity) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[86] +func (x *WorkerResources) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[89] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6182,29 +6370,37 @@ func (x *WorkerCapacity) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use WorkerCapacity.ProtoReflect.Descriptor instead. -func (*WorkerCapacity) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{86} +// Deprecated: Use WorkerResources.ProtoReflect.Descriptor instead. +func (*WorkerResources) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{89} } -func (x *WorkerCapacity) GetCpuMilli() int64 { +func (x *WorkerResources) GetResources() *Resources { if x != nil { - return x.CpuMilli + return x.Resources } - return 0 + return nil } -func (x *WorkerCapacity) GetMemoryBytes() int64 { +func (x *WorkerResources) GetActors() int32 { if x != nil { - return x.MemoryBytes + return x.Actors } return 0 } -// ActorAssignment names the Actor currently bound to a Worker — the inverse of -// WorkerAssignment. +// ActorAssignment is an Actor bound to a Worker, a subresource of Worker +// rather than a field on it: a Worker hosts thousands, and a listing that +// carried them would grow with occupancy. The inverse of WorkerAssignment. type ActorAssignment struct { state protoimpl.MessageState `protogen:"open.v1"` + // atespace is always empty, as it is on the Worker these belong to. name is + // the Actor's UID, so the Actor that caused an assignment is what addresses + // it. + // + // +k8s:required + // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix + Metadata *ResourceMetadata `protobuf:"bytes,6,opt,name=metadata,proto3" json:"metadata,omitempty"` // +k8s:required // +k8s:subfield(atespace)=+k8s:required Actor *ObjectRef `protobuf:"bytes,2,opt,name=actor,proto3" json:"actor,omitempty"` @@ -6217,13 +6413,21 @@ type ActorAssignment struct { // +k8s:required // +k8s:subfield(atespace)=+k8s:required ActorTemplateRef *ObjectRef `protobuf:"bytes,4,opt,name=actor_template_ref,json=actorTemplateRef,proto3" json:"actor_template_ref,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // What the Worker admitted this Actor for: the Actor's declared limits as + // they read at placement time. Recorded rather than re-read so release + // returns exactly what binding consumed, even if the template changed since. + // + // Sorted by name, as WorkerResources.resources is. + // + // +k8s:optional + Resources *Resources `protobuf:"bytes,5,opt,name=resources,proto3" json:"resources,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ActorAssignment) Reset() { *x = ActorAssignment{} - mi := &file_ateapi_proto_msgTypes[87] + mi := &file_ateapi_proto_msgTypes[90] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6235,7 +6439,7 @@ func (x *ActorAssignment) String() string { func (*ActorAssignment) ProtoMessage() {} func (x *ActorAssignment) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[87] + mi := &file_ateapi_proto_msgTypes[90] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6248,7 +6452,14 @@ func (x *ActorAssignment) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorAssignment.ProtoReflect.Descriptor instead. func (*ActorAssignment) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{87} + return file_ateapi_proto_rawDescGZIP(), []int{90} +} + +func (x *ActorAssignment) GetMetadata() *ResourceMetadata { + if x != nil { + return x.Metadata + } + return nil } func (x *ActorAssignment) GetActor() *ObjectRef { @@ -6272,6 +6483,120 @@ func (x *ActorAssignment) GetActorTemplateRef() *ObjectRef { return nil } +func (x *ActorAssignment) GetResources() *Resources { + if x != nil { + return x.Resources + } + return nil +} + +type SetWorkerCapacityRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The Worker being reported on. atespace is always empty; Workers are + // global-scoped. + // + // +k8s:required + // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix + Worker *ObjectRef `protobuf:"bytes,1,opt,name=worker,proto3" json:"worker,omitempty"` + // Everything the Worker can hold. This replaces what is recorded rather than + // merging into it: a dimension left out is one the Worker no longer supplies, + // and an Actor asking for that dimension will not be placed here. + // + // +k8s:required + Capacity *WorkerResources `protobuf:"bytes,2,opt,name=capacity,proto3" json:"capacity,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetWorkerCapacityRequest) Reset() { + *x = SetWorkerCapacityRequest{} + mi := &file_ateapi_proto_msgTypes[91] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetWorkerCapacityRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetWorkerCapacityRequest) ProtoMessage() {} + +func (x *SetWorkerCapacityRequest) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[91] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetWorkerCapacityRequest.ProtoReflect.Descriptor instead. +func (*SetWorkerCapacityRequest) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{91} +} + +func (x *SetWorkerCapacityRequest) GetWorker() *ObjectRef { + if x != nil { + return x.Worker + } + return nil +} + +func (x *SetWorkerCapacityRequest) GetCapacity() *WorkerResources { + if x != nil { + return x.Capacity + } + return nil +} + +type SetWorkerCapacityResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The Worker as recorded, so a caller sees what its report resolved to. + Worker *Worker `protobuf:"bytes,1,opt,name=worker,proto3" json:"worker,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetWorkerCapacityResponse) Reset() { + *x = SetWorkerCapacityResponse{} + mi := &file_ateapi_proto_msgTypes[92] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetWorkerCapacityResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetWorkerCapacityResponse) ProtoMessage() {} + +func (x *SetWorkerCapacityResponse) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[92] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetWorkerCapacityResponse.ProtoReflect.Descriptor instead. +func (*SetWorkerCapacityResponse) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{92} +} + +func (x *SetWorkerCapacityResponse) GetWorker() *Worker { + if x != nil { + return x.Worker + } + return nil +} + type MintJWTRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // The audiences the minted JWT is bound to. Tokens are only issued with @@ -6297,7 +6622,7 @@ type MintJWTRequest struct { func (x *MintJWTRequest) Reset() { *x = MintJWTRequest{} - mi := &file_ateapi_proto_msgTypes[88] + mi := &file_ateapi_proto_msgTypes[93] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6309,7 +6634,7 @@ func (x *MintJWTRequest) String() string { func (*MintJWTRequest) ProtoMessage() {} func (x *MintJWTRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[88] + mi := &file_ateapi_proto_msgTypes[93] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6322,7 +6647,7 @@ func (x *MintJWTRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintJWTRequest.ProtoReflect.Descriptor instead. func (*MintJWTRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{88} + return file_ateapi_proto_rawDescGZIP(), []int{93} } func (x *MintJWTRequest) GetAudience() []string { @@ -6381,7 +6706,7 @@ type MintJWTResponse struct { func (x *MintJWTResponse) Reset() { *x = MintJWTResponse{} - mi := &file_ateapi_proto_msgTypes[89] + mi := &file_ateapi_proto_msgTypes[94] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6393,7 +6718,7 @@ func (x *MintJWTResponse) String() string { func (*MintJWTResponse) ProtoMessage() {} func (x *MintJWTResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[89] + mi := &file_ateapi_proto_msgTypes[94] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6406,7 +6731,7 @@ func (x *MintJWTResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintJWTResponse.ProtoReflect.Descriptor instead. func (*MintJWTResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{89} + return file_ateapi_proto_rawDescGZIP(), []int{94} } func (x *MintJWTResponse) GetActorJwt() string { @@ -6453,7 +6778,7 @@ type MintCertRequest struct { func (x *MintCertRequest) Reset() { *x = MintCertRequest{} - mi := &file_ateapi_proto_msgTypes[90] + mi := &file_ateapi_proto_msgTypes[95] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6465,7 +6790,7 @@ func (x *MintCertRequest) String() string { func (*MintCertRequest) ProtoMessage() {} func (x *MintCertRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[90] + mi := &file_ateapi_proto_msgTypes[95] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6478,7 +6803,7 @@ func (x *MintCertRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintCertRequest.ProtoReflect.Descriptor instead. func (*MintCertRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{90} + return file_ateapi_proto_rawDescGZIP(), []int{95} } func (x *MintCertRequest) GetWorker() *ObjectRef { @@ -6521,7 +6846,7 @@ type MintCertResponse struct { func (x *MintCertResponse) Reset() { *x = MintCertResponse{} - mi := &file_ateapi_proto_msgTypes[91] + mi := &file_ateapi_proto_msgTypes[96] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6533,7 +6858,7 @@ func (x *MintCertResponse) String() string { func (*MintCertResponse) ProtoMessage() {} func (x *MintCertResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[91] + mi := &file_ateapi_proto_msgTypes[96] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6546,7 +6871,7 @@ func (x *MintCertResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintCertResponse.ProtoReflect.Descriptor instead. func (*MintCertResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{91} + return file_ateapi_proto_rawDescGZIP(), []int{96} } func (x *MintCertResponse) GetActorCertificates() [][]byte { @@ -6833,7 +7158,15 @@ const file_ateapi_proto_rawDesc = "" + "\x12actor_snapshot_tag\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x10actorSnapshotTag\";\n" + "\rDeleteOptions\x12\x18\n" + "\aversion\x18\x01 \x01(\x03R\aversion\x12\x10\n" + - "\x03uid\x18\x02 \x01(\tR\x03uid\"P\n" + + "\x03uid\x18\x02 \x01(\tR\x03uid\"\x8a\x01\n" + + "!ListWorkerActorAssignmentsRequest\x12)\n" + + "\x06worker\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x06worker\x12\x1b\n" + + "\tpage_size\x18\x02 \x01(\x05R\bpageSize\x12\x1d\n" + + "\n" + + "page_token\x18\x03 \x01(\tR\tpageToken\"\x92\x01\n" + + "\"ListWorkerActorAssignmentsResponse\x12D\n" + + "\x11actor_assignments\x18\x01 \x03(\v2\x17.ateapi.ActorAssignmentR\x10actorAssignments\x12&\n" + + "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"P\n" + "\x12ListWorkersRequest\x12\x1b\n" + "\tpage_size\x18\x01 \x01(\x05R\bpageSize\x12\x1d\n" + "\n" + @@ -6859,7 +7192,7 @@ const file_ateapi_proto_rawDesc = "" + "page_token\x18\x03 \x01(\tR\tpageToken\"c\n" + "\x12ListActorsResponse\x12%\n" + "\x06actors\x18\x01 \x03(\v2\r.ateapi.ActorR\x06actors\x12&\n" + - "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"\xf2\x03\n" + + "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"\xbe\x03\n" + "\x06Worker\x124\n" + "\bmetadata\x18\x01 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata\x12)\n" + "\x10worker_namespace\x18\x02 \x01(\tR\x0fworkerNamespace\x12\x1f\n" + @@ -6871,25 +7204,33 @@ const file_ateapi_proto_rawDesc = "" + "\tnode_name\x18\x06 \x01(\tR\bnodeName\x12\x0e\n" + "\x02ip\x18\a \x01(\tR\x02ip\x12#\n" + "\rsandbox_class\x18\b \x01(\tR\fsandboxClass\x122\n" + - "\x06labels\x18\t \x03(\v2\x1a.ateapi.Worker.LabelsEntryR\x06labels\x122\n" + - "\bcapacity\x18\n" + - " \x01(\v2\x16.ateapi.WorkerCapacityR\bcapacity\x12,\n" + + "\x06labels\x18\t \x03(\v2\x1a.ateapi.Worker.LabelsEntryR\x06labels\x12,\n" + "\x06status\x18\v \x01(\v2\x14.ateapi.WorkerStatusR\x06status\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"r\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"s\n" + "\fWorkerStatus\x12)\n" + - "\x05state\x18\x01 \x01(\x0e2\x13.ateapi.WorkerStateR\x05state\x127\n" + + "\x05state\x18\x01 \x01(\x0e2\x13.ateapi.WorkerStateR\x05state\x128\n" + "\n" + - "assignment\x18\x02 \x01(\v2\x17.ateapi.ActorAssignmentR\n" + - "assignment\"P\n" + - "\x0eWorkerCapacity\x12\x1b\n" + - "\tcpu_milli\x18\x01 \x01(\x03R\bcpuMilli\x12!\n" + - "\fmemory_bytes\x18\x02 \x01(\x03R\vmemoryBytes\"\x98\x01\n" + - "\x0fActorAssignment\x12'\n" + + "allocation\x18\x02 \x01(\v2\x18.ateapi.WorkerAllocationR\n" + + "allocation\"~\n" + + "\x10WorkerAllocation\x123\n" + + "\bcapacity\x18\x01 \x01(\v2\x17.ateapi.WorkerResourcesR\bcapacity\x125\n" + + "\tallocated\x18\x02 \x01(\v2\x17.ateapi.WorkerResourcesR\tallocated\"Z\n" + + "\x0fWorkerResources\x12/\n" + + "\tresources\x18\x01 \x01(\v2\x11.ateapi.ResourcesR\tresources\x12\x16\n" + + "\x06actors\x18\x02 \x01(\x05R\x06actors\"\xff\x01\n" + + "\x0fActorAssignment\x124\n" + + "\bmetadata\x18\x06 \x01(\v2\x18.ateapi.ResourceMetadataR\bmetadata\x12'\n" + "\x05actor\x18\x02 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x1b\n" + "\tactor_uid\x18\x03 \x01(\tR\bactorUid\x12?\n" + - "\x12actor_template_ref\x18\x04 \x01(\v2\x11.ateapi.ObjectRefR\x10actorTemplateRef\"\x84\x01\n" + + "\x12actor_template_ref\x18\x04 \x01(\v2\x11.ateapi.ObjectRefR\x10actorTemplateRef\x12/\n" + + "\tresources\x18\x05 \x01(\v2\x11.ateapi.ResourcesR\tresources\"z\n" + + "\x18SetWorkerCapacityRequest\x12)\n" + + "\x06worker\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x06worker\x123\n" + + "\bcapacity\x18\x02 \x01(\v2\x17.ateapi.WorkerResourcesR\bcapacity\"C\n" + + "\x19SetWorkerCapacityResponse\x12&\n" + + "\x06worker\x18\x01 \x01(\v2\x0e.ateapi.WorkerR\x06worker\"\x84\x01\n" + "\x0eMintJWTRequest\x12\x1a\n" + "\baudience\x18\x01 \x03(\tR\baudience\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x12\x1d\n" + @@ -6943,7 +7284,7 @@ const file_ateapi_proto_rawDesc = "" + "\x15WORKER_STATE_DRAINING\x10\x02*k\n" + "\x17ActorCertificatePurpose\x12)\n" + "%ACTOR_CERTIFICATE_PURPOSE_UNSPECIFIED\x10\x00\x12%\n" + - "!ACTOR_CERTIFICATE_PURPOSE_ATUNNEL\x10\x012\x84\x13\n" + + "!ACTOR_CERTIFICATE_PURPOSE_ATUNNEL\x10\x012\xfb\x13\n" + "\aControl\x124\n" + "\bGetActor\x12\x17.ateapi.GetActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n" + "\vCreateActor\x12\x1a.ateapi.CreateActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n" + @@ -6968,7 +7309,8 @@ const file_ateapi_proto_rawDesc = "" + "\fCreateWorker\x12\x1b.ateapi.CreateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n" + "\fUpdateWorker\x12\x1b.ateapi.UpdateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n" + "\fDeleteWorker\x12\x1b.ateapi.DeleteWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12;\n" + - "\vDrainWorker\x12\x1a.ateapi.DrainWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12E\n" + + "\vDrainWorker\x12\x1a.ateapi.DrainWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12u\n" + + "\x1aListWorkerActorAssignments\x12).ateapi.ListWorkerActorAssignmentsRequest\x1a*.ateapi.ListWorkerActorAssignmentsResponse\"\x00\x12E\n" + "\n" + "ListActors\x12\x19.ateapi.ListActorsRequest\x1a\x1a.ateapi.ListActorsResponse\"\x00\x12C\n" + "\x0eCreateAtespace\x12\x1d.ateapi.CreateAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12=\n" + @@ -6981,7 +7323,9 @@ const file_ateapi_proto_rawDesc = "" + "\x13DeleteActorTemplate\x12\".ateapi.DeleteActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x002\x8a\x01\n" + "\rActorIdentity\x12:\n" + "\aMintJWT\x12\x16.ateapi.MintJWTRequest\x1a\x17.ateapi.MintJWTResponse\x12=\n" + - "\bMintCert\x12\x17.ateapi.MintCertRequest\x1a\x18.ateapi.MintCertResponseB9Z7github.com/agent-substrate/substrate/pkg/proto/ateapipbb\x06proto3" + "\bMintCert\x12\x17.ateapi.MintCertRequest\x1a\x18.ateapi.MintCertResponse2i\n" + + "\rWorkerService\x12X\n" + + "\x11SetWorkerCapacity\x12 .ateapi.SetWorkerCapacityRequest\x1a!.ateapi.SetWorkerCapacityResponseB9Z7github.com/agent-substrate/substrate/pkg/proto/ateapipbb\x06proto3" var ( file_ateapi_proto_rawDescOnce sync.Once @@ -6996,122 +7340,127 @@ func file_ateapi_proto_rawDescGZIP() []byte { } var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 9) -var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 95) +var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 100) var file_ateapi_proto_goTypes = []any{ - (SnapshotContentScope)(0), // 0: ateapi.SnapshotContentScope - (ActorSnapshotTagScope)(0), // 1: ateapi.ActorSnapshotTagScope - (ActorState)(0), // 2: ateapi.ActorState - (SandboxClass)(0), // 3: ateapi.SandboxClass - (ResumeSource)(0), // 4: ateapi.ResumeSource - (ActorMetadataField)(0), // 5: ateapi.ActorMetadataField - (WorkerState)(0), // 6: ateapi.WorkerState - (ActorCertificatePurpose)(0), // 7: ateapi.ActorCertificatePurpose - (ExternalVolume_Status)(0), // 8: ateapi.ExternalVolume.Status - (*LocalSnapshotInfo)(nil), // 9: ateapi.LocalSnapshotInfo - (*Selector)(nil), // 10: ateapi.Selector - (*ResourceMetadata)(nil), // 11: ateapi.ResourceMetadata - (*ExternalVolume)(nil), // 12: ateapi.ExternalVolume - (*Actor)(nil), // 13: ateapi.Actor - (*EgressPolicy)(nil), // 14: ateapi.EgressPolicy - (*EgressRule)(nil), // 15: ateapi.EgressRule - (*HostnameRule)(nil), // 16: ateapi.HostnameRule - (*IPBlockRule)(nil), // 17: ateapi.IPBlockRule - (*EgressRuleEffects)(nil), // 18: ateapi.EgressRuleEffects - (*CredentialHeaderInjection)(nil), // 19: ateapi.CredentialHeaderInjection - (*ActorStatus)(nil), // 20: ateapi.ActorStatus - (*ActorSourceSnapshotStatus)(nil), // 21: ateapi.ActorSourceSnapshotStatus - (*WorkerAssignment)(nil), // 22: ateapi.WorkerAssignment - (*ActorSnapshot)(nil), // 23: ateapi.ActorSnapshot - (*ActorSnapshotStatus)(nil), // 24: ateapi.ActorSnapshotStatus - (*ActorSnapshotTag)(nil), // 25: ateapi.ActorSnapshotTag - (*Atespace)(nil), // 26: ateapi.Atespace - (*ObjectRef)(nil), // 27: ateapi.ObjectRef - (*ActorTemplate)(nil), // 28: ateapi.ActorTemplate - (*Resources)(nil), // 29: ateapi.Resources - (*Limits)(nil), // 30: ateapi.Limits - (*GoldenSnapshotStatus)(nil), // 31: ateapi.GoldenSnapshotStatus - (*ActorTemplateStatus)(nil), // 32: ateapi.ActorTemplateStatus - (*SandboxConfig)(nil), // 33: ateapi.SandboxConfig - (*SnapshotsConfig)(nil), // 34: ateapi.SnapshotsConfig - (*OnResumeConfig)(nil), // 35: ateapi.OnResumeConfig - (*Container)(nil), // 36: ateapi.Container - (*SecurityContext)(nil), // 37: ateapi.SecurityContext - (*Capabilities)(nil), // 38: ateapi.Capabilities - (*EnvVar)(nil), // 39: ateapi.EnvVar - (*ContainerReadyz)(nil), // 40: ateapi.ContainerReadyz - (*HTTPGetAction)(nil), // 41: ateapi.HTTPGetAction - (*Volume)(nil), // 42: ateapi.Volume - (*ImageVolumeSource)(nil), // 43: ateapi.ImageVolumeSource - (*DurableDirVolumeSource)(nil), // 44: ateapi.DurableDirVolumeSource - (*ExternalVolumeTemplate)(nil), // 45: ateapi.ExternalVolumeTemplate - (*SystemInfoVolumeSource)(nil), // 46: ateapi.SystemInfoVolumeSource - (*SystemInfoDataSource)(nil), // 47: ateapi.SystemInfoDataSource - (*ActorMetadataDataSource)(nil), // 48: ateapi.ActorMetadataDataSource - (*ActorMetadataItem)(nil), // 49: ateapi.ActorMetadataItem - (*TrustBundleDataSource)(nil), // 50: ateapi.TrustBundleDataSource - (*VolumeMount)(nil), // 51: ateapi.VolumeMount - (*CreateAtespaceRequest)(nil), // 52: ateapi.CreateAtespaceRequest - (*GetAtespaceRequest)(nil), // 53: ateapi.GetAtespaceRequest - (*ListAtespacesRequest)(nil), // 54: ateapi.ListAtespacesRequest - (*ListAtespacesResponse)(nil), // 55: ateapi.ListAtespacesResponse - (*DeleteAtespaceRequest)(nil), // 56: ateapi.DeleteAtespaceRequest - (*CreateActorTemplateRequest)(nil), // 57: ateapi.CreateActorTemplateRequest - (*GetActorTemplateRequest)(nil), // 58: ateapi.GetActorTemplateRequest - (*ListActorTemplatesRequest)(nil), // 59: ateapi.ListActorTemplatesRequest - (*ListActorTemplatesResponse)(nil), // 60: ateapi.ListActorTemplatesResponse - (*DeleteActorTemplateRequest)(nil), // 61: ateapi.DeleteActorTemplateRequest - (*GetActorRequest)(nil), // 62: ateapi.GetActorRequest - (*CreateActorRequest)(nil), // 63: ateapi.CreateActorRequest - (*UpdateActorRequest)(nil), // 64: ateapi.UpdateActorRequest - (*SuspendActorRequest)(nil), // 65: ateapi.SuspendActorRequest - (*SuspendActorResponse)(nil), // 66: ateapi.SuspendActorResponse - (*PauseActorRequest)(nil), // 67: ateapi.PauseActorRequest - (*PauseActorResponse)(nil), // 68: ateapi.PauseActorResponse - (*ResumeActorRequest)(nil), // 69: ateapi.ResumeActorRequest - (*ResumeActorResponse)(nil), // 70: ateapi.ResumeActorResponse - (*DeleteActorRequest)(nil), // 71: ateapi.DeleteActorRequest - (*GetActorEgressPolicyRequest)(nil), // 72: ateapi.GetActorEgressPolicyRequest - (*CreateActorEgressPolicyRequest)(nil), // 73: ateapi.CreateActorEgressPolicyRequest - (*UpdateActorEgressPolicyRequest)(nil), // 74: ateapi.UpdateActorEgressPolicyRequest - (*DeleteActorEgressPolicyRequest)(nil), // 75: ateapi.DeleteActorEgressPolicyRequest - (*GetActorSnapshotRequest)(nil), // 76: ateapi.GetActorSnapshotRequest - (*GetActorSnapshotTagRequest)(nil), // 77: ateapi.GetActorSnapshotTagRequest - (*ListActorSnapshotsRequest)(nil), // 78: ateapi.ListActorSnapshotsRequest - (*ListActorSnapshotsResponse)(nil), // 79: ateapi.ListActorSnapshotsResponse - (*CreateActorSnapshotTagRequest)(nil), // 80: ateapi.CreateActorSnapshotTagRequest - (*UpdateActorSnapshotTagRequest)(nil), // 81: ateapi.UpdateActorSnapshotTagRequest - (*DeleteActorSnapshotTagRequest)(nil), // 82: ateapi.DeleteActorSnapshotTagRequest - (*DeleteOptions)(nil), // 83: ateapi.DeleteOptions - (*ListWorkersRequest)(nil), // 84: ateapi.ListWorkersRequest - (*ListWorkersResponse)(nil), // 85: ateapi.ListWorkersResponse - (*GetWorkerRequest)(nil), // 86: ateapi.GetWorkerRequest - (*CreateWorkerRequest)(nil), // 87: ateapi.CreateWorkerRequest - (*UpdateWorkerRequest)(nil), // 88: ateapi.UpdateWorkerRequest - (*DeleteWorkerRequest)(nil), // 89: ateapi.DeleteWorkerRequest - (*DrainWorkerRequest)(nil), // 90: ateapi.DrainWorkerRequest - (*ListActorsRequest)(nil), // 91: ateapi.ListActorsRequest - (*ListActorsResponse)(nil), // 92: ateapi.ListActorsResponse - (*Worker)(nil), // 93: ateapi.Worker - (*WorkerStatus)(nil), // 94: ateapi.WorkerStatus - (*WorkerCapacity)(nil), // 95: ateapi.WorkerCapacity - (*ActorAssignment)(nil), // 96: ateapi.ActorAssignment - (*MintJWTRequest)(nil), // 97: ateapi.MintJWTRequest - (*MintJWTResponse)(nil), // 98: ateapi.MintJWTResponse - (*MintCertRequest)(nil), // 99: ateapi.MintCertRequest - (*MintCertResponse)(nil), // 100: ateapi.MintCertResponse - nil, // 101: ateapi.Selector.MatchLabelsEntry - nil, // 102: ateapi.ExternalVolume.VolumeContextEntry - nil, // 103: ateapi.Worker.LabelsEntry - (*timestamppb.Timestamp)(nil), // 104: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 105: google.protobuf.Empty + (SnapshotContentScope)(0), // 0: ateapi.SnapshotContentScope + (ActorSnapshotTagScope)(0), // 1: ateapi.ActorSnapshotTagScope + (ActorState)(0), // 2: ateapi.ActorState + (SandboxClass)(0), // 3: ateapi.SandboxClass + (ResumeSource)(0), // 4: ateapi.ResumeSource + (ActorMetadataField)(0), // 5: ateapi.ActorMetadataField + (WorkerState)(0), // 6: ateapi.WorkerState + (ActorCertificatePurpose)(0), // 7: ateapi.ActorCertificatePurpose + (ExternalVolume_Status)(0), // 8: ateapi.ExternalVolume.Status + (*LocalSnapshotInfo)(nil), // 9: ateapi.LocalSnapshotInfo + (*Selector)(nil), // 10: ateapi.Selector + (*ResourceMetadata)(nil), // 11: ateapi.ResourceMetadata + (*ExternalVolume)(nil), // 12: ateapi.ExternalVolume + (*Actor)(nil), // 13: ateapi.Actor + (*EgressPolicy)(nil), // 14: ateapi.EgressPolicy + (*EgressRule)(nil), // 15: ateapi.EgressRule + (*HostnameRule)(nil), // 16: ateapi.HostnameRule + (*IPBlockRule)(nil), // 17: ateapi.IPBlockRule + (*EgressRuleEffects)(nil), // 18: ateapi.EgressRuleEffects + (*CredentialHeaderInjection)(nil), // 19: ateapi.CredentialHeaderInjection + (*ActorStatus)(nil), // 20: ateapi.ActorStatus + (*ActorSourceSnapshotStatus)(nil), // 21: ateapi.ActorSourceSnapshotStatus + (*WorkerAssignment)(nil), // 22: ateapi.WorkerAssignment + (*ActorSnapshot)(nil), // 23: ateapi.ActorSnapshot + (*ActorSnapshotStatus)(nil), // 24: ateapi.ActorSnapshotStatus + (*ActorSnapshotTag)(nil), // 25: ateapi.ActorSnapshotTag + (*Atespace)(nil), // 26: ateapi.Atespace + (*ObjectRef)(nil), // 27: ateapi.ObjectRef + (*ActorTemplate)(nil), // 28: ateapi.ActorTemplate + (*Resources)(nil), // 29: ateapi.Resources + (*Limits)(nil), // 30: ateapi.Limits + (*GoldenSnapshotStatus)(nil), // 31: ateapi.GoldenSnapshotStatus + (*ActorTemplateStatus)(nil), // 32: ateapi.ActorTemplateStatus + (*SandboxConfig)(nil), // 33: ateapi.SandboxConfig + (*SnapshotsConfig)(nil), // 34: ateapi.SnapshotsConfig + (*OnResumeConfig)(nil), // 35: ateapi.OnResumeConfig + (*Container)(nil), // 36: ateapi.Container + (*SecurityContext)(nil), // 37: ateapi.SecurityContext + (*Capabilities)(nil), // 38: ateapi.Capabilities + (*EnvVar)(nil), // 39: ateapi.EnvVar + (*ContainerReadyz)(nil), // 40: ateapi.ContainerReadyz + (*HTTPGetAction)(nil), // 41: ateapi.HTTPGetAction + (*Volume)(nil), // 42: ateapi.Volume + (*ImageVolumeSource)(nil), // 43: ateapi.ImageVolumeSource + (*DurableDirVolumeSource)(nil), // 44: ateapi.DurableDirVolumeSource + (*ExternalVolumeTemplate)(nil), // 45: ateapi.ExternalVolumeTemplate + (*SystemInfoVolumeSource)(nil), // 46: ateapi.SystemInfoVolumeSource + (*SystemInfoDataSource)(nil), // 47: ateapi.SystemInfoDataSource + (*ActorMetadataDataSource)(nil), // 48: ateapi.ActorMetadataDataSource + (*ActorMetadataItem)(nil), // 49: ateapi.ActorMetadataItem + (*TrustBundleDataSource)(nil), // 50: ateapi.TrustBundleDataSource + (*VolumeMount)(nil), // 51: ateapi.VolumeMount + (*CreateAtespaceRequest)(nil), // 52: ateapi.CreateAtespaceRequest + (*GetAtespaceRequest)(nil), // 53: ateapi.GetAtespaceRequest + (*ListAtespacesRequest)(nil), // 54: ateapi.ListAtespacesRequest + (*ListAtespacesResponse)(nil), // 55: ateapi.ListAtespacesResponse + (*DeleteAtespaceRequest)(nil), // 56: ateapi.DeleteAtespaceRequest + (*CreateActorTemplateRequest)(nil), // 57: ateapi.CreateActorTemplateRequest + (*GetActorTemplateRequest)(nil), // 58: ateapi.GetActorTemplateRequest + (*ListActorTemplatesRequest)(nil), // 59: ateapi.ListActorTemplatesRequest + (*ListActorTemplatesResponse)(nil), // 60: ateapi.ListActorTemplatesResponse + (*DeleteActorTemplateRequest)(nil), // 61: ateapi.DeleteActorTemplateRequest + (*GetActorRequest)(nil), // 62: ateapi.GetActorRequest + (*CreateActorRequest)(nil), // 63: ateapi.CreateActorRequest + (*UpdateActorRequest)(nil), // 64: ateapi.UpdateActorRequest + (*SuspendActorRequest)(nil), // 65: ateapi.SuspendActorRequest + (*SuspendActorResponse)(nil), // 66: ateapi.SuspendActorResponse + (*PauseActorRequest)(nil), // 67: ateapi.PauseActorRequest + (*PauseActorResponse)(nil), // 68: ateapi.PauseActorResponse + (*ResumeActorRequest)(nil), // 69: ateapi.ResumeActorRequest + (*ResumeActorResponse)(nil), // 70: ateapi.ResumeActorResponse + (*DeleteActorRequest)(nil), // 71: ateapi.DeleteActorRequest + (*GetActorEgressPolicyRequest)(nil), // 72: ateapi.GetActorEgressPolicyRequest + (*CreateActorEgressPolicyRequest)(nil), // 73: ateapi.CreateActorEgressPolicyRequest + (*UpdateActorEgressPolicyRequest)(nil), // 74: ateapi.UpdateActorEgressPolicyRequest + (*DeleteActorEgressPolicyRequest)(nil), // 75: ateapi.DeleteActorEgressPolicyRequest + (*GetActorSnapshotRequest)(nil), // 76: ateapi.GetActorSnapshotRequest + (*GetActorSnapshotTagRequest)(nil), // 77: ateapi.GetActorSnapshotTagRequest + (*ListActorSnapshotsRequest)(nil), // 78: ateapi.ListActorSnapshotsRequest + (*ListActorSnapshotsResponse)(nil), // 79: ateapi.ListActorSnapshotsResponse + (*CreateActorSnapshotTagRequest)(nil), // 80: ateapi.CreateActorSnapshotTagRequest + (*UpdateActorSnapshotTagRequest)(nil), // 81: ateapi.UpdateActorSnapshotTagRequest + (*DeleteActorSnapshotTagRequest)(nil), // 82: ateapi.DeleteActorSnapshotTagRequest + (*DeleteOptions)(nil), // 83: ateapi.DeleteOptions + (*ListWorkerActorAssignmentsRequest)(nil), // 84: ateapi.ListWorkerActorAssignmentsRequest + (*ListWorkerActorAssignmentsResponse)(nil), // 85: ateapi.ListWorkerActorAssignmentsResponse + (*ListWorkersRequest)(nil), // 86: ateapi.ListWorkersRequest + (*ListWorkersResponse)(nil), // 87: ateapi.ListWorkersResponse + (*GetWorkerRequest)(nil), // 88: ateapi.GetWorkerRequest + (*CreateWorkerRequest)(nil), // 89: ateapi.CreateWorkerRequest + (*UpdateWorkerRequest)(nil), // 90: ateapi.UpdateWorkerRequest + (*DeleteWorkerRequest)(nil), // 91: ateapi.DeleteWorkerRequest + (*DrainWorkerRequest)(nil), // 92: ateapi.DrainWorkerRequest + (*ListActorsRequest)(nil), // 93: ateapi.ListActorsRequest + (*ListActorsResponse)(nil), // 94: ateapi.ListActorsResponse + (*Worker)(nil), // 95: ateapi.Worker + (*WorkerStatus)(nil), // 96: ateapi.WorkerStatus + (*WorkerAllocation)(nil), // 97: ateapi.WorkerAllocation + (*WorkerResources)(nil), // 98: ateapi.WorkerResources + (*ActorAssignment)(nil), // 99: ateapi.ActorAssignment + (*SetWorkerCapacityRequest)(nil), // 100: ateapi.SetWorkerCapacityRequest + (*SetWorkerCapacityResponse)(nil), // 101: ateapi.SetWorkerCapacityResponse + (*MintJWTRequest)(nil), // 102: ateapi.MintJWTRequest + (*MintJWTResponse)(nil), // 103: ateapi.MintJWTResponse + (*MintCertRequest)(nil), // 104: ateapi.MintCertRequest + (*MintCertResponse)(nil), // 105: ateapi.MintCertResponse + nil, // 106: ateapi.Selector.MatchLabelsEntry + nil, // 107: ateapi.ExternalVolume.VolumeContextEntry + nil, // 108: ateapi.Worker.LabelsEntry + (*timestamppb.Timestamp)(nil), // 109: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 110: google.protobuf.Empty } var file_ateapi_proto_depIdxs = []int32{ 0, // 0: ateapi.LocalSnapshotInfo.content_scope:type_name -> ateapi.SnapshotContentScope - 101, // 1: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry - 104, // 2: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp - 104, // 3: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 106, // 1: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry + 109, // 2: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 109, // 3: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp 8, // 4: ateapi.ExternalVolume.status:type_name -> ateapi.ExternalVolume.Status - 102, // 5: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry + 107, // 5: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry 11, // 6: ateapi.Actor.metadata:type_name -> ateapi.ResourceMetadata 27, // 7: ateapi.Actor.actor_template:type_name -> ateapi.ObjectRef 10, // 8: ateapi.Actor.worker_selector:type_name -> ateapi.Selector @@ -7121,7 +7470,7 @@ var file_ateapi_proto_depIdxs = []int32{ 15, // 12: ateapi.EgressPolicy.rules:type_name -> ateapi.EgressRule 16, // 13: ateapi.EgressRule.hostnames:type_name -> ateapi.HostnameRule 17, // 14: ateapi.EgressRule.ip_blocks:type_name -> ateapi.IPBlockRule - 105, // 15: ateapi.EgressRule.all:type_name -> google.protobuf.Empty + 110, // 15: ateapi.EgressRule.all:type_name -> google.protobuf.Empty 18, // 16: ateapi.HostnameRule.effects:type_name -> ateapi.EgressRuleEffects 19, // 17: ateapi.EgressRuleEffects.inject_static_headers:type_name -> ateapi.CredentialHeaderInjection 2, // 18: ateapi.ActorStatus.state:type_name -> ateapi.ActorState @@ -7152,7 +7501,7 @@ var file_ateapi_proto_depIdxs = []int32{ 32, // 43: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus 30, // 44: ateapi.Resources.limits:type_name -> ateapi.Limits 27, // 45: ateapi.GoldenSnapshotStatus.golden_snapshot:type_name -> ateapi.ObjectRef - 104, // 46: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp + 109, // 46: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp 31, // 47: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus 3, // 48: ateapi.SandboxConfig.sandbox_class:type_name -> ateapi.SandboxClass 0, // 49: ateapi.SnapshotsConfig.on_pause:type_name -> ateapi.SnapshotContentScope @@ -7205,97 +7554,110 @@ var file_ateapi_proto_depIdxs = []int32{ 25, // 96: ateapi.CreateActorSnapshotTagRequest.actor_snapshot_tag:type_name -> ateapi.ActorSnapshotTag 25, // 97: ateapi.UpdateActorSnapshotTagRequest.actor_snapshot_tag:type_name -> ateapi.ActorSnapshotTag 27, // 98: ateapi.DeleteActorSnapshotTagRequest.actor_snapshot_tag:type_name -> ateapi.ObjectRef - 93, // 99: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker - 27, // 100: ateapi.GetWorkerRequest.worker:type_name -> ateapi.ObjectRef - 93, // 101: ateapi.CreateWorkerRequest.worker:type_name -> ateapi.Worker - 93, // 102: ateapi.UpdateWorkerRequest.worker:type_name -> ateapi.Worker - 27, // 103: ateapi.DeleteWorkerRequest.worker:type_name -> ateapi.ObjectRef - 83, // 104: ateapi.DeleteWorkerRequest.options:type_name -> ateapi.DeleteOptions - 27, // 105: ateapi.DrainWorkerRequest.worker:type_name -> ateapi.ObjectRef - 13, // 106: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor - 11, // 107: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata - 103, // 108: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry - 95, // 109: ateapi.Worker.capacity:type_name -> ateapi.WorkerCapacity - 94, // 110: ateapi.Worker.status:type_name -> ateapi.WorkerStatus - 6, // 111: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState - 96, // 112: ateapi.WorkerStatus.assignment:type_name -> ateapi.ActorAssignment - 27, // 113: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef - 27, // 114: ateapi.ActorAssignment.actor_template_ref:type_name -> ateapi.ObjectRef - 27, // 115: ateapi.MintCertRequest.worker:type_name -> ateapi.ObjectRef - 7, // 116: ateapi.MintCertRequest.purpose:type_name -> ateapi.ActorCertificatePurpose - 62, // 117: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest - 63, // 118: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest - 64, // 119: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest - 65, // 120: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest - 67, // 121: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest - 69, // 122: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest - 71, // 123: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest - 72, // 124: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest - 73, // 125: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest - 74, // 126: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest - 75, // 127: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest - 76, // 128: ateapi.Control.GetActorSnapshot:input_type -> ateapi.GetActorSnapshotRequest - 77, // 129: ateapi.Control.GetActorSnapshotTag:input_type -> ateapi.GetActorSnapshotTagRequest - 78, // 130: ateapi.Control.ListActorSnapshots:input_type -> ateapi.ListActorSnapshotsRequest - 80, // 131: ateapi.Control.CreateActorSnapshotTag:input_type -> ateapi.CreateActorSnapshotTagRequest - 81, // 132: ateapi.Control.UpdateActorSnapshotTag:input_type -> ateapi.UpdateActorSnapshotTagRequest - 82, // 133: ateapi.Control.DeleteActorSnapshotTag:input_type -> ateapi.DeleteActorSnapshotTagRequest - 84, // 134: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest - 86, // 135: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest - 87, // 136: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest - 88, // 137: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest - 89, // 138: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest - 90, // 139: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest - 91, // 140: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest - 52, // 141: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest - 53, // 142: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest - 54, // 143: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest - 56, // 144: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest - 57, // 145: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest - 58, // 146: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest - 59, // 147: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest - 61, // 148: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest - 97, // 149: ateapi.ActorIdentity.MintJWT:input_type -> ateapi.MintJWTRequest - 99, // 150: ateapi.ActorIdentity.MintCert:input_type -> ateapi.MintCertRequest - 13, // 151: ateapi.Control.GetActor:output_type -> ateapi.Actor - 13, // 152: ateapi.Control.CreateActor:output_type -> ateapi.Actor - 13, // 153: ateapi.Control.UpdateActor:output_type -> ateapi.Actor - 66, // 154: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse - 68, // 155: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse - 70, // 156: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse - 13, // 157: ateapi.Control.DeleteActor:output_type -> ateapi.Actor - 14, // 158: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy - 14, // 159: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 14, // 160: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 14, // 161: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy - 23, // 162: ateapi.Control.GetActorSnapshot:output_type -> ateapi.ActorSnapshot - 25, // 163: ateapi.Control.GetActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag - 79, // 164: ateapi.Control.ListActorSnapshots:output_type -> ateapi.ListActorSnapshotsResponse - 25, // 165: ateapi.Control.CreateActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag - 25, // 166: ateapi.Control.UpdateActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag - 25, // 167: ateapi.Control.DeleteActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag - 85, // 168: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse - 93, // 169: ateapi.Control.GetWorker:output_type -> ateapi.Worker - 93, // 170: ateapi.Control.CreateWorker:output_type -> ateapi.Worker - 93, // 171: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker - 93, // 172: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker - 93, // 173: ateapi.Control.DrainWorker:output_type -> ateapi.Worker - 92, // 174: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse - 26, // 175: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace - 26, // 176: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace - 55, // 177: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse - 26, // 178: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace - 28, // 179: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate - 28, // 180: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate - 60, // 181: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse - 28, // 182: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate - 98, // 183: ateapi.ActorIdentity.MintJWT:output_type -> ateapi.MintJWTResponse - 100, // 184: ateapi.ActorIdentity.MintCert:output_type -> ateapi.MintCertResponse - 151, // [151:185] is the sub-list for method output_type - 117, // [117:151] is the sub-list for method input_type - 117, // [117:117] is the sub-list for extension type_name - 117, // [117:117] is the sub-list for extension extendee - 0, // [0:117] is the sub-list for field type_name + 27, // 99: ateapi.ListWorkerActorAssignmentsRequest.worker:type_name -> ateapi.ObjectRef + 99, // 100: ateapi.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> ateapi.ActorAssignment + 95, // 101: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker + 27, // 102: ateapi.GetWorkerRequest.worker:type_name -> ateapi.ObjectRef + 95, // 103: ateapi.CreateWorkerRequest.worker:type_name -> ateapi.Worker + 95, // 104: ateapi.UpdateWorkerRequest.worker:type_name -> ateapi.Worker + 27, // 105: ateapi.DeleteWorkerRequest.worker:type_name -> ateapi.ObjectRef + 83, // 106: ateapi.DeleteWorkerRequest.options:type_name -> ateapi.DeleteOptions + 27, // 107: ateapi.DrainWorkerRequest.worker:type_name -> ateapi.ObjectRef + 13, // 108: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor + 11, // 109: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata + 108, // 110: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry + 96, // 111: ateapi.Worker.status:type_name -> ateapi.WorkerStatus + 6, // 112: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState + 97, // 113: ateapi.WorkerStatus.allocation:type_name -> ateapi.WorkerAllocation + 98, // 114: ateapi.WorkerAllocation.capacity:type_name -> ateapi.WorkerResources + 98, // 115: ateapi.WorkerAllocation.allocated:type_name -> ateapi.WorkerResources + 29, // 116: ateapi.WorkerResources.resources:type_name -> ateapi.Resources + 11, // 117: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata + 27, // 118: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef + 27, // 119: ateapi.ActorAssignment.actor_template_ref:type_name -> ateapi.ObjectRef + 29, // 120: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources + 27, // 121: ateapi.SetWorkerCapacityRequest.worker:type_name -> ateapi.ObjectRef + 98, // 122: ateapi.SetWorkerCapacityRequest.capacity:type_name -> ateapi.WorkerResources + 95, // 123: ateapi.SetWorkerCapacityResponse.worker:type_name -> ateapi.Worker + 27, // 124: ateapi.MintCertRequest.worker:type_name -> ateapi.ObjectRef + 7, // 125: ateapi.MintCertRequest.purpose:type_name -> ateapi.ActorCertificatePurpose + 62, // 126: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest + 63, // 127: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest + 64, // 128: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest + 65, // 129: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest + 67, // 130: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest + 69, // 131: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest + 71, // 132: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest + 72, // 133: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest + 73, // 134: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest + 74, // 135: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest + 75, // 136: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest + 76, // 137: ateapi.Control.GetActorSnapshot:input_type -> ateapi.GetActorSnapshotRequest + 77, // 138: ateapi.Control.GetActorSnapshotTag:input_type -> ateapi.GetActorSnapshotTagRequest + 78, // 139: ateapi.Control.ListActorSnapshots:input_type -> ateapi.ListActorSnapshotsRequest + 80, // 140: ateapi.Control.CreateActorSnapshotTag:input_type -> ateapi.CreateActorSnapshotTagRequest + 81, // 141: ateapi.Control.UpdateActorSnapshotTag:input_type -> ateapi.UpdateActorSnapshotTagRequest + 82, // 142: ateapi.Control.DeleteActorSnapshotTag:input_type -> ateapi.DeleteActorSnapshotTagRequest + 86, // 143: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest + 88, // 144: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest + 89, // 145: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest + 90, // 146: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest + 91, // 147: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest + 92, // 148: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest + 84, // 149: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest + 93, // 150: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest + 52, // 151: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest + 53, // 152: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest + 54, // 153: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest + 56, // 154: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest + 57, // 155: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest + 58, // 156: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest + 59, // 157: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest + 61, // 158: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest + 102, // 159: ateapi.ActorIdentity.MintJWT:input_type -> ateapi.MintJWTRequest + 104, // 160: ateapi.ActorIdentity.MintCert:input_type -> ateapi.MintCertRequest + 100, // 161: ateapi.WorkerService.SetWorkerCapacity:input_type -> ateapi.SetWorkerCapacityRequest + 13, // 162: ateapi.Control.GetActor:output_type -> ateapi.Actor + 13, // 163: ateapi.Control.CreateActor:output_type -> ateapi.Actor + 13, // 164: ateapi.Control.UpdateActor:output_type -> ateapi.Actor + 66, // 165: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse + 68, // 166: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse + 70, // 167: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse + 13, // 168: ateapi.Control.DeleteActor:output_type -> ateapi.Actor + 14, // 169: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy + 14, // 170: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 14, // 171: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 14, // 172: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy + 23, // 173: ateapi.Control.GetActorSnapshot:output_type -> ateapi.ActorSnapshot + 25, // 174: ateapi.Control.GetActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag + 79, // 175: ateapi.Control.ListActorSnapshots:output_type -> ateapi.ListActorSnapshotsResponse + 25, // 176: ateapi.Control.CreateActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag + 25, // 177: ateapi.Control.UpdateActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag + 25, // 178: ateapi.Control.DeleteActorSnapshotTag:output_type -> ateapi.ActorSnapshotTag + 87, // 179: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse + 95, // 180: ateapi.Control.GetWorker:output_type -> ateapi.Worker + 95, // 181: ateapi.Control.CreateWorker:output_type -> ateapi.Worker + 95, // 182: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker + 95, // 183: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker + 95, // 184: ateapi.Control.DrainWorker:output_type -> ateapi.Worker + 85, // 185: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse + 94, // 186: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse + 26, // 187: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace + 26, // 188: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace + 55, // 189: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse + 26, // 190: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace + 28, // 191: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate + 28, // 192: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate + 60, // 193: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse + 28, // 194: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate + 103, // 195: ateapi.ActorIdentity.MintJWT:output_type -> ateapi.MintJWTResponse + 105, // 196: ateapi.ActorIdentity.MintCert:output_type -> ateapi.MintCertResponse + 101, // 197: ateapi.WorkerService.SetWorkerCapacity:output_type -> ateapi.SetWorkerCapacityResponse + 162, // [162:198] is the sub-list for method output_type + 126, // [126:162] is the sub-list for method input_type + 126, // [126:126] is the sub-list for extension type_name + 126, // [126:126] is the sub-list for extension extendee + 0, // [0:126] is the sub-list for field type_name } func init() { file_ateapi_proto_init() } @@ -7309,9 +7671,9 @@ func file_ateapi_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateapi_proto_rawDesc), len(file_ateapi_proto_rawDesc)), NumEnums: 9, - NumMessages: 95, + NumMessages: 100, NumExtensions: 0, - NumServices: 2, + NumServices: 3, }, GoTypes: file_ateapi_proto_goTypes, DependencyIndexes: file_ateapi_proto_depIdxs, diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index ed94c0aac1..27019a518f 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -98,6 +98,10 @@ service Control { // Returns ABORTED if another write lands on the Worker first; retry. rpc DrainWorker(DrainWorkerRequest) returns (Worker) {} + // List the Actors a Worker hosts. A subresource of Worker rather than a field + // on it, so GetWorker and ListWorkers cost the same whatever the occupancy. + rpc ListWorkerActorAssignments(ListWorkerActorAssignmentsRequest) returns (ListWorkerActorAssignmentsResponse) {} + // List Actors. rpc ListActors(ListActorsRequest) returns (ListActorsResponse) {} @@ -1480,6 +1484,39 @@ message DeleteOptions { string uid = 2; } +// ListWorkerActorAssignmentsRequest asks for a page of the Actors one Worker hosts. +message ListWorkerActorAssignmentsRequest { + // The Worker whose Actors to list. atespace is always empty; Workers are + // global-scoped. + // + // +k8s:required + // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix + ObjectRef worker = 1; + + // Requested page size; the server may return fewer. If unspecified, defaults + // to a server-chosen value; values above 1000 are coerced to 1000. + // + // +k8s:optional + // +k8s:minimum=1 + int32 page_size = 2; + + // Pagination token from a previous ListWorkerActorAssignments response. + // Omit or leave empty for the first request. + // + // +k8s:optional + // +k8s:maxLength=256 + string page_token = 3; +} + +// ListWorkerActorAssignmentsResponse is one page of a Worker's Actors. +message ListWorkerActorAssignmentsResponse { + // The Actors this page of the listing covers. + repeated ActorAssignment actor_assignments = 1; + + // Pagination token for the next page. Empty if this is the last page. + string next_page_token = 2; +} + message ListWorkersRequest { // Requested page size; the server may return fewer, or occasionally // slightly more. If unspecified, defaults to a server-chosen value; @@ -1661,18 +1698,9 @@ message Worker { // +k8s:eachVal=+k8s:format=k8s-label-value map labels = 9; - // The compute capacity this worker can give an actor sandbox. Immutable, set - // at creation: a worker pod's limits are fixed for its lifetime. An update - // replaces the worker rather than patching it, so a request that omits - // capacity is asking to clear it, which the immutability rule rejects. - // - // +k8s:optional - // +k8s:immutable - WorkerCapacity capacity = 10; - // Output-only server-managed state. Absent from Create/Update request // payloads; whatever a request carries here is ignored. DrainWorker is the - // only way a client moves state, and assignment is the scheduler's. + // only way a client moves state, and the assignments are the scheduler's. // // +k8s:optional WorkerStatus status = 11; @@ -1693,35 +1721,70 @@ message WorkerStatus { // +k8s:maximum=2 # keep this in sync with the WorkerState enum WorkerState state = 1; - // The Actor currently bound to this Worker, if any. + // What this Worker has and what is spoken for. Observed, not requested: no + // client sets it, which is why it is status. // // +k8s:optional - ActorAssignment assignment = 2; + WorkerAllocation allocation = 2; } -// WorkerCapacity is the worker pod's compute capacity available to host an -// actor sandbox, taken from the ateom container's resource limits. The -// scheduler only places an actor on a worker whose capacity is >= the actor's -// declared resource limits. An unset message, or a zero field within it, means -// "unknown/unset" for that dimension: treated as unconstrained so placement is -// not blocked (matching the pre-capacity behavior). -message WorkerCapacity { - // CPU capacity in millicores (1000 = one core). +// WorkerAllocation is a Worker's total against its committed share, in one +// message because placement only ever reads the two together. +message WorkerAllocation { + // What the Worker can supply, as it reports through + // WorkerService.SetWorkerCapacity. + // + // Complete, not partial: a Worker reports every dimension it has, so a name + // missing here is one it cannot supply at all and no Actor asking for that + // name is placed on it. Shrinking below allocated stops new placements and + // evicts nothing. // // +k8s:optional - // +k8s:minimum=1 - int64 cpu_milli = 1; + WorkerResources capacity = 1; - // Memory capacity in bytes. + // What the assignments consume of capacity. A running total because placement + // reads it for every Worker on every decision, and summing the assignments + // would cost the fleet's actor count each time. + // + // +k8s:optional + WorkerResources allocated = 2; +} + +// WorkerResources represents schedulable resources. +message WorkerResources { + // Resources match the actor's definition of resources. + // They are matched by name, used for placement. + // + // Sorted by name, so equal capacities compare equal. + // + // +k8s:optional + Resources resources = 1; + + // How many Actors may be bound at once. Not a resource because nothing + // divides it: every Actor costs exactly one. It bounds what one more Actor + // costs where CPU and memory do not — netns, mounts, file descriptors, blast + // radius. Kubernetes bounds a node the same way, with allocatable pods. + // + // Absent until the Worker reports, and a Worker that has reported no ceiling + // holds no Actors, like any other dimension. // // +k8s:optional // +k8s:minimum=1 - int64 memory_bytes = 2; + int32 actors = 2; } -// ActorAssignment names the Actor currently bound to a Worker — the inverse of -// WorkerAssignment. +// ActorAssignment is an Actor bound to a Worker, a subresource of Worker +// rather than a field on it: a Worker hosts thousands, and a listing that +// carried them would grow with occupancy. The inverse of WorkerAssignment. message ActorAssignment { + // atespace is always empty, as it is on the Worker these belong to. name is + // the Actor's UID, so the Actor that caused an assignment is what addresses + // it. + // + // +k8s:required + // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix + ResourceMetadata metadata = 6; + // +k8s:required // +k8s:subfield(atespace)=+k8s:required ObjectRef actor = 2; @@ -1736,6 +1799,15 @@ message ActorAssignment { // +k8s:required // +k8s:subfield(atespace)=+k8s:required ObjectRef actor_template_ref = 4; + + // What the Worker admitted this Actor for: the Actor's declared limits as + // they read at placement time. Recorded rather than re-read so release + // returns exactly what binding consumed, even if the template changed since. + // + // Sorted by name, as WorkerResources.resources is. + // + // +k8s:optional + Resources resources = 5; } // ActorIdentity allows substrate workloads to exchange their @@ -1765,6 +1837,42 @@ service ActorIdentity { rpc MintCert(MintCertRequest) returns (MintCertResponse); } +// WorkerService is how a Worker tells the control plane about itself. It is +// separate from Control because the two have different callers and different +// authorization: Control is the client-facing API, while these RPCs are served +// only to an atelet, and only for the Workers on its own node. +service WorkerService { + // SetWorkerCapacity records what a Worker can hold. Capacity is the Worker's + // to report rather than the control plane's to infer: it is what the ateom + // can actually supply, only its node can observe it, and a fleet may run + // mixed ateom versions. + // + // atelet calls this with its own client certificate, as it does for + // MintCert. Idempotent: re-sending the same capacity is not a write. + rpc SetWorkerCapacity(SetWorkerCapacityRequest) returns (SetWorkerCapacityResponse); +} + +message SetWorkerCapacityRequest { + // The Worker being reported on. atespace is always empty; Workers are + // global-scoped. + // + // +k8s:required + // +k8s:beta(since: "0.0")=+k8s:subfield(atespace)=+k8s:forbidden # TODO: get rid of beta prefix + ObjectRef worker = 1; + + // Everything the Worker can hold. This replaces what is recorded rather than + // merging into it: a dimension left out is one the Worker no longer supplies, + // and an Actor asking for that dimension will not be placed here. + // + // +k8s:required + WorkerResources capacity = 2; +} + +message SetWorkerCapacityResponse { + // The Worker as recorded, so a caller sees what its report resolved to. + Worker worker = 1; +} + message MintJWTRequest { // The audiences the minted JWT is bound to. Tokens are only issued with // audience bindings, so at least one is required. diff --git a/pkg/proto/ateapipb/ateapi_grpc.pb.go b/pkg/proto/ateapipb/ateapi_grpc.pb.go index 14cb7af62f..793d99c624 100644 --- a/pkg/proto/ateapipb/ateapi_grpc.pb.go +++ b/pkg/proto/ateapipb/ateapi_grpc.pb.go @@ -33,38 +33,39 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Control_GetActor_FullMethodName = "/ateapi.Control/GetActor" - Control_CreateActor_FullMethodName = "/ateapi.Control/CreateActor" - Control_UpdateActor_FullMethodName = "/ateapi.Control/UpdateActor" - Control_SuspendActor_FullMethodName = "/ateapi.Control/SuspendActor" - Control_PauseActor_FullMethodName = "/ateapi.Control/PauseActor" - Control_ResumeActor_FullMethodName = "/ateapi.Control/ResumeActor" - Control_DeleteActor_FullMethodName = "/ateapi.Control/DeleteActor" - Control_GetActorEgressPolicy_FullMethodName = "/ateapi.Control/GetActorEgressPolicy" - Control_CreateActorEgressPolicy_FullMethodName = "/ateapi.Control/CreateActorEgressPolicy" - Control_UpdateActorEgressPolicy_FullMethodName = "/ateapi.Control/UpdateActorEgressPolicy" - Control_DeleteActorEgressPolicy_FullMethodName = "/ateapi.Control/DeleteActorEgressPolicy" - Control_GetActorSnapshot_FullMethodName = "/ateapi.Control/GetActorSnapshot" - Control_GetActorSnapshotTag_FullMethodName = "/ateapi.Control/GetActorSnapshotTag" - Control_ListActorSnapshots_FullMethodName = "/ateapi.Control/ListActorSnapshots" - Control_CreateActorSnapshotTag_FullMethodName = "/ateapi.Control/CreateActorSnapshotTag" - Control_UpdateActorSnapshotTag_FullMethodName = "/ateapi.Control/UpdateActorSnapshotTag" - Control_DeleteActorSnapshotTag_FullMethodName = "/ateapi.Control/DeleteActorSnapshotTag" - Control_ListWorkers_FullMethodName = "/ateapi.Control/ListWorkers" - Control_GetWorker_FullMethodName = "/ateapi.Control/GetWorker" - Control_CreateWorker_FullMethodName = "/ateapi.Control/CreateWorker" - Control_UpdateWorker_FullMethodName = "/ateapi.Control/UpdateWorker" - Control_DeleteWorker_FullMethodName = "/ateapi.Control/DeleteWorker" - Control_DrainWorker_FullMethodName = "/ateapi.Control/DrainWorker" - Control_ListActors_FullMethodName = "/ateapi.Control/ListActors" - Control_CreateAtespace_FullMethodName = "/ateapi.Control/CreateAtespace" - Control_GetAtespace_FullMethodName = "/ateapi.Control/GetAtespace" - Control_ListAtespaces_FullMethodName = "/ateapi.Control/ListAtespaces" - Control_DeleteAtespace_FullMethodName = "/ateapi.Control/DeleteAtespace" - Control_CreateActorTemplate_FullMethodName = "/ateapi.Control/CreateActorTemplate" - Control_GetActorTemplate_FullMethodName = "/ateapi.Control/GetActorTemplate" - Control_ListActorTemplates_FullMethodName = "/ateapi.Control/ListActorTemplates" - Control_DeleteActorTemplate_FullMethodName = "/ateapi.Control/DeleteActorTemplate" + Control_GetActor_FullMethodName = "/ateapi.Control/GetActor" + Control_CreateActor_FullMethodName = "/ateapi.Control/CreateActor" + Control_UpdateActor_FullMethodName = "/ateapi.Control/UpdateActor" + Control_SuspendActor_FullMethodName = "/ateapi.Control/SuspendActor" + Control_PauseActor_FullMethodName = "/ateapi.Control/PauseActor" + Control_ResumeActor_FullMethodName = "/ateapi.Control/ResumeActor" + Control_DeleteActor_FullMethodName = "/ateapi.Control/DeleteActor" + Control_GetActorEgressPolicy_FullMethodName = "/ateapi.Control/GetActorEgressPolicy" + Control_CreateActorEgressPolicy_FullMethodName = "/ateapi.Control/CreateActorEgressPolicy" + Control_UpdateActorEgressPolicy_FullMethodName = "/ateapi.Control/UpdateActorEgressPolicy" + Control_DeleteActorEgressPolicy_FullMethodName = "/ateapi.Control/DeleteActorEgressPolicy" + Control_GetActorSnapshot_FullMethodName = "/ateapi.Control/GetActorSnapshot" + Control_GetActorSnapshotTag_FullMethodName = "/ateapi.Control/GetActorSnapshotTag" + Control_ListActorSnapshots_FullMethodName = "/ateapi.Control/ListActorSnapshots" + Control_CreateActorSnapshotTag_FullMethodName = "/ateapi.Control/CreateActorSnapshotTag" + Control_UpdateActorSnapshotTag_FullMethodName = "/ateapi.Control/UpdateActorSnapshotTag" + Control_DeleteActorSnapshotTag_FullMethodName = "/ateapi.Control/DeleteActorSnapshotTag" + Control_ListWorkers_FullMethodName = "/ateapi.Control/ListWorkers" + Control_GetWorker_FullMethodName = "/ateapi.Control/GetWorker" + Control_CreateWorker_FullMethodName = "/ateapi.Control/CreateWorker" + Control_UpdateWorker_FullMethodName = "/ateapi.Control/UpdateWorker" + Control_DeleteWorker_FullMethodName = "/ateapi.Control/DeleteWorker" + Control_DrainWorker_FullMethodName = "/ateapi.Control/DrainWorker" + Control_ListWorkerActorAssignments_FullMethodName = "/ateapi.Control/ListWorkerActorAssignments" + Control_ListActors_FullMethodName = "/ateapi.Control/ListActors" + Control_CreateAtespace_FullMethodName = "/ateapi.Control/CreateAtespace" + Control_GetAtespace_FullMethodName = "/ateapi.Control/GetAtespace" + Control_ListAtespaces_FullMethodName = "/ateapi.Control/ListAtespaces" + Control_DeleteAtespace_FullMethodName = "/ateapi.Control/DeleteAtespace" + Control_CreateActorTemplate_FullMethodName = "/ateapi.Control/CreateActorTemplate" + Control_GetActorTemplate_FullMethodName = "/ateapi.Control/GetActorTemplate" + Control_ListActorTemplates_FullMethodName = "/ateapi.Control/ListActorTemplates" + Control_DeleteActorTemplate_FullMethodName = "/ateapi.Control/DeleteActorTemplate" ) // ControlClient is the client API for Control service. @@ -125,6 +126,9 @@ type ControlClient interface { // it. Idempotent; one-way. Deliberately leaves any bound Actor alone. // Returns ABORTED if another write lands on the Worker first; retry. DrainWorker(ctx context.Context, in *DrainWorkerRequest, opts ...grpc.CallOption) (*Worker, error) + // List the Actors a Worker hosts. A subresource of Worker rather than a field + // on it, so GetWorker and ListWorkers cost the same whatever the occupancy. + ListWorkerActorAssignments(ctx context.Context, in *ListWorkerActorAssignmentsRequest, opts ...grpc.CallOption) (*ListWorkerActorAssignmentsResponse, error) // List Actors. ListActors(ctx context.Context, in *ListActorsRequest, opts ...grpc.CallOption) (*ListActorsResponse, error) // Create a new Atespace. Substrate-native, stored in database. @@ -382,6 +386,16 @@ func (c *controlClient) DrainWorker(ctx context.Context, in *DrainWorkerRequest, return out, nil } +func (c *controlClient) ListWorkerActorAssignments(ctx context.Context, in *ListWorkerActorAssignmentsRequest, opts ...grpc.CallOption) (*ListWorkerActorAssignmentsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListWorkerActorAssignmentsResponse) + err := c.cc.Invoke(ctx, Control_ListWorkerActorAssignments_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *controlClient) ListActors(ctx context.Context, in *ListActorsRequest, opts ...grpc.CallOption) (*ListActorsResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ListActorsResponse) @@ -530,6 +544,9 @@ type ControlServer interface { // it. Idempotent; one-way. Deliberately leaves any bound Actor alone. // Returns ABORTED if another write lands on the Worker first; retry. DrainWorker(context.Context, *DrainWorkerRequest) (*Worker, error) + // List the Actors a Worker hosts. A subresource of Worker rather than a field + // on it, so GetWorker and ListWorkers cost the same whatever the occupancy. + ListWorkerActorAssignments(context.Context, *ListWorkerActorAssignmentsRequest) (*ListWorkerActorAssignmentsResponse, error) // List Actors. ListActors(context.Context, *ListActorsRequest) (*ListActorsResponse, error) // Create a new Atespace. Substrate-native, stored in database. @@ -626,6 +643,9 @@ func (UnimplementedControlServer) DeleteWorker(context.Context, *DeleteWorkerReq func (UnimplementedControlServer) DrainWorker(context.Context, *DrainWorkerRequest) (*Worker, error) { return nil, status.Error(codes.Unimplemented, "method DrainWorker not implemented") } +func (UnimplementedControlServer) ListWorkerActorAssignments(context.Context, *ListWorkerActorAssignmentsRequest) (*ListWorkerActorAssignmentsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListWorkerActorAssignments not implemented") +} func (UnimplementedControlServer) ListActors(context.Context, *ListActorsRequest) (*ListActorsResponse, error) { return nil, status.Error(codes.Unimplemented, "method ListActors not implemented") } @@ -1088,6 +1108,24 @@ func _Control_DrainWorker_Handler(srv interface{}, ctx context.Context, dec func return interceptor(ctx, in, info, handler) } +func _Control_ListWorkerActorAssignments_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListWorkerActorAssignmentsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).ListWorkerActorAssignments(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_ListWorkerActorAssignments_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).ListWorkerActorAssignments(ctx, req.(*ListWorkerActorAssignmentsRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Control_ListActors_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ListActorsRequest) if err := dec(in); err != nil { @@ -1349,6 +1387,10 @@ var Control_ServiceDesc = grpc.ServiceDesc{ MethodName: "DrainWorker", Handler: _Control_DrainWorker_Handler, }, + { + MethodName: "ListWorkerActorAssignments", + Handler: _Control_ListWorkerActorAssignments_Handler, + }, { MethodName: "ListActors", Handler: _Control_ListActors_Handler, @@ -1573,3 +1615,129 @@ var ActorIdentity_ServiceDesc = grpc.ServiceDesc{ Streams: []grpc.StreamDesc{}, Metadata: "ateapi.proto", } + +const ( + WorkerService_SetWorkerCapacity_FullMethodName = "/ateapi.WorkerService/SetWorkerCapacity" +) + +// WorkerServiceClient is the client API for WorkerService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// WorkerService is how a Worker tells the control plane about itself. It is +// separate from Control because the two have different callers and different +// authorization: Control is the client-facing API, while these RPCs are served +// only to an atelet, and only for the Workers on its own node. +type WorkerServiceClient interface { + // SetWorkerCapacity records what a Worker can hold. Capacity is the Worker's + // to report rather than the control plane's to infer: it is what the ateom + // can actually supply, only its node can observe it, and a fleet may run + // mixed ateom versions. + // + // atelet calls this with its own client certificate, as it does for + // MintCert. Idempotent: re-sending the same capacity is not a write. + SetWorkerCapacity(ctx context.Context, in *SetWorkerCapacityRequest, opts ...grpc.CallOption) (*SetWorkerCapacityResponse, error) +} + +type workerServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewWorkerServiceClient(cc grpc.ClientConnInterface) WorkerServiceClient { + return &workerServiceClient{cc} +} + +func (c *workerServiceClient) SetWorkerCapacity(ctx context.Context, in *SetWorkerCapacityRequest, opts ...grpc.CallOption) (*SetWorkerCapacityResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SetWorkerCapacityResponse) + err := c.cc.Invoke(ctx, WorkerService_SetWorkerCapacity_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// WorkerServiceServer is the server API for WorkerService service. +// All implementations must embed UnimplementedWorkerServiceServer +// for forward compatibility. +// +// WorkerService is how a Worker tells the control plane about itself. It is +// separate from Control because the two have different callers and different +// authorization: Control is the client-facing API, while these RPCs are served +// only to an atelet, and only for the Workers on its own node. +type WorkerServiceServer interface { + // SetWorkerCapacity records what a Worker can hold. Capacity is the Worker's + // to report rather than the control plane's to infer: it is what the ateom + // can actually supply, only its node can observe it, and a fleet may run + // mixed ateom versions. + // + // atelet calls this with its own client certificate, as it does for + // MintCert. Idempotent: re-sending the same capacity is not a write. + SetWorkerCapacity(context.Context, *SetWorkerCapacityRequest) (*SetWorkerCapacityResponse, error) + mustEmbedUnimplementedWorkerServiceServer() +} + +// UnimplementedWorkerServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedWorkerServiceServer struct{} + +func (UnimplementedWorkerServiceServer) SetWorkerCapacity(context.Context, *SetWorkerCapacityRequest) (*SetWorkerCapacityResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SetWorkerCapacity not implemented") +} +func (UnimplementedWorkerServiceServer) mustEmbedUnimplementedWorkerServiceServer() {} +func (UnimplementedWorkerServiceServer) testEmbeddedByValue() {} + +// UnsafeWorkerServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to WorkerServiceServer will +// result in compilation errors. +type UnsafeWorkerServiceServer interface { + mustEmbedUnimplementedWorkerServiceServer() +} + +func RegisterWorkerServiceServer(s grpc.ServiceRegistrar, srv WorkerServiceServer) { + // If the following call panics, it indicates UnimplementedWorkerServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&WorkerService_ServiceDesc, srv) +} + +func _WorkerService_SetWorkerCapacity_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetWorkerCapacityRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerServiceServer).SetWorkerCapacity(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: WorkerService_SetWorkerCapacity_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerServiceServer).SetWorkerCapacity(ctx, req.(*SetWorkerCapacityRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// WorkerService_ServiceDesc is the grpc.ServiceDesc for WorkerService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var WorkerService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "ateapi.WorkerService", + HandlerType: (*WorkerServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "SetWorkerCapacity", + Handler: _WorkerService_SetWorkerCapacity_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "ateapi.proto", +} diff --git a/tools/apitool/exemptions.json b/tools/apitool/exemptions.json index 652b7e73cd..b5a564a673 100644 --- a/tools/apitool/exemptions.json +++ b/tools/apitool/exemptions.json @@ -553,5 +553,15 @@ "rule": "documented", "subject": "ateapi.WorkerStatus.state", "message": "field has no doc comment" + }, + { + "rule": "list-method-shape", + "subject": "ateapi.Control.ListWorkerActorAssignments", + "message": "field \"worker\" is not part of the standard List request shape - only atespace, page_size, and page_token are allowed (sorting/filtering per AIP-132 is not supported)" + }, + { + "rule": "list-method-shape", + "subject": "ateapi.Control.ListWorkerActorAssignments", + "message": "repeated field is named \"actor_assignments\", want \"worker_actor_assignments\" (the plural resource name, matching the method name)" } ]