diff --git a/Makefile b/Makefile index 943393d1..53e92d72 100644 --- a/Makefile +++ b/Makefile @@ -229,9 +229,9 @@ test-linux: ensure-ch-binaries ensure-caddy-binaries build-embedded if [ -n "$(VERBOSE)" ]; then VERBOSE_FLAG="-v"; fi; \ if [ -n "$(TEST)" ]; then \ echo "Running specific test: $(TEST)"; \ - sudo env "PATH=$$PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=180s ./...; \ + sudo env "PATH=$$PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=300s ./...; \ else \ - sudo env "PATH=$$PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=180s ./...; \ + sudo env "PATH=$$PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=300s ./...; \ fi # macOS tests (no sudo needed, adds e2fsprogs to PATH) @@ -246,10 +246,10 @@ test-darwin: build-embedded if [ -n "$(TEST)" ]; then \ echo "Running specific test: $(TEST)"; \ PATH="/opt/homebrew/opt/e2fsprogs/sbin:$(PATH)" \ - go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=180s $$PKGS; \ + go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=300s $$PKGS; \ else \ PATH="/opt/homebrew/opt/e2fsprogs/sbin:$(PATH)" \ - go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=180s $$PKGS; \ + go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=300s $$PKGS; \ fi # Generate JWT token for testing diff --git a/cmd/api/api/exec_test.go b/cmd/api/api/exec_test.go index be21bc5c..1edf5eba 100644 --- a/cmd/api/api/exec_test.go +++ b/cmd/api/api/exec_test.go @@ -179,13 +179,17 @@ func TestExecWithDebianMinimal(t *testing.T) { // Create Debian 12 slim image (minimal, no iproute2) createAndWaitForImage(t, svc, "docker.io/library/debian:12-slim", 60*time.Second) - // Create instance (network disabled in test environment) + // Create instance with a long-running command so the VM stays alive for exec. + // Debian's default CMD is "bash" which exits immediately (no stdin), + // and init shuts down the VM after the entrypoint exits. t.Log("Creating Debian instance...") networkEnabled := false + cmdOverride := []string{"sleep", "infinity"} instResp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ Body: &oapi.CreateInstanceRequest{ Name: "debian-exec-test", Image: "docker.io/library/debian:12-slim", + Cmd: &cmdOverride, Network: &struct { BandwidthDownload *string `json:"bandwidth_download,omitempty"` BandwidthUpload *string `json:"bandwidth_upload,omitempty"` @@ -238,11 +242,7 @@ func TestExecWithDebianMinimal(t *testing.T) { } } - // Verify the app exited but VM is still usable (key behavior this test validates) - logs = collectTestLogs(t, svc, inst.Id, 200) - assert.Contains(t, logs, "[exec] app exited with code", "App should have exited") - - // Test exec commands work even though the main app (bash) has exited + // Test exec commands work while the app is running dialer2, err := hypervisor.NewVsockDialer(actualInst.HypervisorType, actualInst.VsockSocket, actualInst.VsockCID) require.NoError(t, err) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index ac8fe5b3..32826665 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -219,6 +219,16 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst } } + // Parse command overrides (like docker run --entrypoint / docker run ) + var entrypoint []string + if request.Body.Entrypoint != nil { + entrypoint = *request.Body.Entrypoint + } + var cmd []string + if request.Body.Cmd != nil { + cmd = *request.Body.Cmd + } + domainReq := instances.CreateInstanceRequest{ Name: request.Body.Name, Image: request.Body.Image, @@ -236,6 +246,8 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Volumes: volumes, Hypervisor: hvType, GPU: gpuConfig, + Entrypoint: entrypoint, + Cmd: cmd, SkipKernelHeaders: request.Body.SkipKernelHeaders != nil && *request.Body.SkipKernelHeaders, SkipGuestAgent: request.Body.SkipGuestAgent != nil && *request.Body.SkipGuestAgent, } @@ -467,7 +479,18 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan } log := logger.FromContext(ctx) - result, err := s.InstanceManager.StartInstance(ctx, inst.Id) + // Parse optional command overrides from request body + var startReq instances.StartInstanceRequest + if request.Body != nil { + if request.Body.Entrypoint != nil { + startReq.Entrypoint = *request.Body.Entrypoint + } + if request.Body.Cmd != nil { + startReq.Cmd = *request.Body.Cmd + } + } + + result, err := s.InstanceManager.StartInstance(ctx, inst.Id, startReq) if err != nil { switch { case errors.Is(err, instances.ErrInvalidState): @@ -732,10 +755,15 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { CreatedAt: inst.CreatedAt, StartedAt: inst.StartedAt, StoppedAt: inst.StoppedAt, + ExitCode: inst.ExitCode, HasSnapshot: lo.ToPtr(inst.HasSnapshot), Hypervisor: &hvType, } + if inst.ExitMessage != "" { + oapiInst.ExitMessage = lo.ToPtr(inst.ExitMessage) + } + if len(inst.Env) > 0 { oapiInst.Env = &inst.Env } diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index 74e148e4..b8863006 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -99,7 +99,7 @@ func (m *mockInstanceManager) StopInstance(ctx context.Context, id string) (*ins return nil, instances.ErrNotFound } -func (m *mockInstanceManager) StartInstance(ctx context.Context, id string) (*instances.Instance, error) { +func (m *mockInstanceManager) StartInstance(ctx context.Context, id string, req instances.StartInstanceRequest) (*instances.Instance, error) { return nil, nil } diff --git a/lib/guest/client.go b/lib/guest/client.go index 0afd7f2a..bee9f3a0 100644 --- a/lib/guest/client.go +++ b/lib/guest/client.go @@ -658,3 +658,23 @@ func CopyFromInstance(ctx context.Context, dialer hypervisor.VsockDialer, opts C } return nil } + +// ShutdownInstance sends a shutdown signal to the guest VM's init process (PID 1). +// The guest-agent forwards the signal to init, which forwards it to the entrypoint. +// sig is the signal number to send (0 = SIGTERM default). +func ShutdownInstance(ctx context.Context, dialer hypervisor.VsockDialer, sig int32) error { + grpcConn, err := GetOrCreateConn(ctx, dialer) + if err != nil { + return fmt.Errorf("get grpc connection: %w", err) + } + + client := NewGuestServiceClient(grpcConn) + _, err = client.Shutdown(ctx, &ShutdownRequest{ + Signal: sig, + }) + if err != nil { + return fmt.Errorf("shutdown RPC: %w", err) + } + + return nil +} diff --git a/lib/guest/guest.pb.go b/lib/guest/guest.pb.go index baf60375..0de1628e 100644 --- a/lib/guest/guest.pb.go +++ b/lib/guest/guest.pb.go @@ -1184,6 +1184,88 @@ func (x *StatPathResponse) GetError() string { return "" } +// ShutdownRequest requests graceful VM shutdown +type ShutdownRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Signal int32 `protobuf:"varint,1,opt,name=signal,proto3" json:"signal,omitempty"` // Signal to send to init (PID 1), 0 = SIGTERM (default) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ShutdownRequest) Reset() { + *x = ShutdownRequest{} + mi := &file_lib_guest_guest_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ShutdownRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ShutdownRequest) ProtoMessage() {} + +func (x *ShutdownRequest) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ShutdownRequest.ProtoReflect.Descriptor instead. +func (*ShutdownRequest) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{15} +} + +func (x *ShutdownRequest) GetSignal() int32 { + if x != nil { + return x.Signal + } + return 0 +} + +// ShutdownResponse acknowledges the shutdown request +type ShutdownResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ShutdownResponse) Reset() { + *x = ShutdownResponse{} + mi := &file_lib_guest_guest_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ShutdownResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ShutdownResponse) ProtoMessage() {} + +func (x *ShutdownResponse) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ShutdownResponse.ProtoReflect.Descriptor instead. +func (*ShutdownResponse) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{16} +} + var File_lib_guest_guest_proto protoreflect.FileDescriptor const file_lib_guest_guest_proto_rawDesc = "" + @@ -1273,12 +1355,16 @@ const file_lib_guest_guest_proto_rawDesc = "" + "linkTarget\x12\x12\n" + "\x04mode\x18\x06 \x01(\rR\x04mode\x12\x12\n" + "\x04size\x18\a \x01(\x03R\x04size\x12\x14\n" + - "\x05error\x18\b \x01(\tR\x05error2\x96\x02\n" + + "\x05error\x18\b \x01(\tR\x05error\")\n" + + "\x0fShutdownRequest\x12\x16\n" + + "\x06signal\x18\x01 \x01(\x05R\x06signal\"\x12\n" + + "\x10ShutdownResponse2\xd3\x02\n" + "\fGuestService\x123\n" + "\x04Exec\x12\x12.guest.ExecRequest\x1a\x13.guest.ExecResponse(\x010\x01\x12F\n" + "\vCopyToGuest\x12\x19.guest.CopyToGuestRequest\x1a\x1a.guest.CopyToGuestResponse(\x01\x12L\n" + "\rCopyFromGuest\x12\x1b.guest.CopyFromGuestRequest\x1a\x1c.guest.CopyFromGuestResponse0\x01\x12;\n" + - "\bStatPath\x12\x16.guest.StatPathRequest\x1a\x17.guest.StatPathResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" + "\bStatPath\x12\x16.guest.StatPathRequest\x1a\x17.guest.StatPathResponse\x12;\n" + + "\bShutdown\x12\x16.guest.ShutdownRequest\x1a\x17.guest.ShutdownResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" var ( file_lib_guest_guest_proto_rawDescOnce sync.Once @@ -1292,7 +1378,7 @@ func file_lib_guest_guest_proto_rawDescGZIP() []byte { return file_lib_guest_guest_proto_rawDescData } -var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 16) +var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 18) var file_lib_guest_guest_proto_goTypes = []any{ (*ExecRequest)(nil), // 0: guest.ExecRequest (*ExecStart)(nil), // 1: guest.ExecStart @@ -1309,12 +1395,14 @@ var file_lib_guest_guest_proto_goTypes = []any{ (*CopyFromGuestError)(nil), // 12: guest.CopyFromGuestError (*StatPathRequest)(nil), // 13: guest.StatPathRequest (*StatPathResponse)(nil), // 14: guest.StatPathResponse - nil, // 15: guest.ExecStart.EnvEntry + (*ShutdownRequest)(nil), // 15: guest.ShutdownRequest + (*ShutdownResponse)(nil), // 16: guest.ShutdownResponse + nil, // 17: guest.ExecStart.EnvEntry } var file_lib_guest_guest_proto_depIdxs = []int32{ 1, // 0: guest.ExecRequest.start:type_name -> guest.ExecStart 2, // 1: guest.ExecRequest.resize:type_name -> guest.WindowSize - 15, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry + 17, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry 5, // 3: guest.CopyToGuestRequest.start:type_name -> guest.CopyToGuestStart 6, // 4: guest.CopyToGuestRequest.end:type_name -> guest.CopyToGuestEnd 10, // 5: guest.CopyFromGuestResponse.header:type_name -> guest.CopyFromGuestHeader @@ -1324,12 +1412,14 @@ var file_lib_guest_guest_proto_depIdxs = []int32{ 4, // 9: guest.GuestService.CopyToGuest:input_type -> guest.CopyToGuestRequest 8, // 10: guest.GuestService.CopyFromGuest:input_type -> guest.CopyFromGuestRequest 13, // 11: guest.GuestService.StatPath:input_type -> guest.StatPathRequest - 3, // 12: guest.GuestService.Exec:output_type -> guest.ExecResponse - 7, // 13: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse - 9, // 14: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse - 14, // 15: guest.GuestService.StatPath:output_type -> guest.StatPathResponse - 12, // [12:16] is the sub-list for method output_type - 8, // [8:12] is the sub-list for method input_type + 15, // 12: guest.GuestService.Shutdown:input_type -> guest.ShutdownRequest + 3, // 13: guest.GuestService.Exec:output_type -> guest.ExecResponse + 7, // 14: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse + 9, // 15: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse + 14, // 16: guest.GuestService.StatPath:output_type -> guest.StatPathResponse + 16, // 17: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse + 13, // [13:18] is the sub-list for method output_type + 8, // [8:13] is the sub-list for method input_type 8, // [8:8] is the sub-list for extension type_name 8, // [8:8] is the sub-list for extension extendee 0, // [0:8] is the sub-list for field type_name @@ -1367,7 +1457,7 @@ func file_lib_guest_guest_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_lib_guest_guest_proto_rawDesc), len(file_lib_guest_guest_proto_rawDesc)), NumEnums: 0, - NumMessages: 16, + NumMessages: 18, NumExtensions: 0, NumServices: 1, }, diff --git a/lib/guest/guest.proto b/lib/guest/guest.proto index db08b316..c42198a9 100644 --- a/lib/guest/guest.proto +++ b/lib/guest/guest.proto @@ -17,6 +17,9 @@ service GuestService { // StatPath returns information about a path in the guest filesystem rpc StatPath(StatPathRequest) returns (StatPathResponse); + + // Shutdown requests graceful VM shutdown by signaling init (PID 1) + rpc Shutdown(ShutdownRequest) returns (ShutdownResponse); } // ExecRequest represents messages from client to server @@ -143,3 +146,11 @@ message StatPathResponse { int64 size = 7; // File size string error = 8; // Error message if stat failed (e.g., permission denied) } + +// ShutdownRequest requests graceful VM shutdown +message ShutdownRequest { + int32 signal = 1; // Signal to send to init (PID 1), 0 = SIGTERM (default) +} + +// ShutdownResponse acknowledges the shutdown request +message ShutdownResponse {} diff --git a/lib/guest/guest_grpc.pb.go b/lib/guest/guest_grpc.pb.go index d656cb60..71224327 100644 --- a/lib/guest/guest_grpc.pb.go +++ b/lib/guest/guest_grpc.pb.go @@ -23,6 +23,7 @@ const ( GuestService_CopyToGuest_FullMethodName = "/guest.GuestService/CopyToGuest" GuestService_CopyFromGuest_FullMethodName = "/guest.GuestService/CopyFromGuest" GuestService_StatPath_FullMethodName = "/guest.GuestService/StatPath" + GuestService_Shutdown_FullMethodName = "/guest.GuestService/Shutdown" ) // GuestServiceClient is the client API for GuestService service. @@ -39,6 +40,8 @@ type GuestServiceClient interface { CopyFromGuest(ctx context.Context, in *CopyFromGuestRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[CopyFromGuestResponse], error) // StatPath returns information about a path in the guest filesystem StatPath(ctx context.Context, in *StatPathRequest, opts ...grpc.CallOption) (*StatPathResponse, error) + // Shutdown requests graceful VM shutdown by signaling init (PID 1) + Shutdown(ctx context.Context, in *ShutdownRequest, opts ...grpc.CallOption) (*ShutdownResponse, error) } type guestServiceClient struct { @@ -104,6 +107,16 @@ func (c *guestServiceClient) StatPath(ctx context.Context, in *StatPathRequest, return out, nil } +func (c *guestServiceClient) Shutdown(ctx context.Context, in *ShutdownRequest, opts ...grpc.CallOption) (*ShutdownResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ShutdownResponse) + err := c.cc.Invoke(ctx, GuestService_Shutdown_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // GuestServiceServer is the server API for GuestService service. // All implementations must embed UnimplementedGuestServiceServer // for forward compatibility. @@ -118,6 +131,8 @@ type GuestServiceServer interface { CopyFromGuest(*CopyFromGuestRequest, grpc.ServerStreamingServer[CopyFromGuestResponse]) error // StatPath returns information about a path in the guest filesystem StatPath(context.Context, *StatPathRequest) (*StatPathResponse, error) + // Shutdown requests graceful VM shutdown by signaling init (PID 1) + Shutdown(context.Context, *ShutdownRequest) (*ShutdownResponse, error) mustEmbedUnimplementedGuestServiceServer() } @@ -140,6 +155,9 @@ func (UnimplementedGuestServiceServer) CopyFromGuest(*CopyFromGuestRequest, grpc func (UnimplementedGuestServiceServer) StatPath(context.Context, *StatPathRequest) (*StatPathResponse, error) { return nil, status.Error(codes.Unimplemented, "method StatPath not implemented") } +func (UnimplementedGuestServiceServer) Shutdown(context.Context, *ShutdownRequest) (*ShutdownResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Shutdown not implemented") +} func (UnimplementedGuestServiceServer) mustEmbedUnimplementedGuestServiceServer() {} func (UnimplementedGuestServiceServer) testEmbeddedByValue() {} @@ -204,6 +222,24 @@ func _GuestService_StatPath_Handler(srv interface{}, ctx context.Context, dec fu return interceptor(ctx, in, info, handler) } +func _GuestService_Shutdown_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ShutdownRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GuestServiceServer).Shutdown(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GuestService_Shutdown_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GuestServiceServer).Shutdown(ctx, req.(*ShutdownRequest)) + } + return interceptor(ctx, in, info, handler) +} + // GuestService_ServiceDesc is the grpc.ServiceDesc for GuestService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -215,6 +251,10 @@ var GuestService_ServiceDesc = grpc.ServiceDesc{ MethodName: "StatPath", Handler: _GuestService_StatPath_Handler, }, + { + MethodName: "Shutdown", + Handler: _GuestService_Shutdown_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/lib/instances/configdisk.go b/lib/instances/configdisk.go index 7d3f73c3..4e17051c 100644 --- a/lib/instances/configdisk.go +++ b/lib/instances/configdisk.go @@ -47,9 +47,20 @@ func (m *manager) createConfigDisk(ctx context.Context, inst *Instance, imageInf // buildGuestConfig creates the vmconfig.Config struct for the guest init binary. func (m *manager) buildGuestConfig(ctx context.Context, inst *Instance, imageInfo *images.Image, netConfig *network.NetworkConfig) *vmconfig.Config { + // Use instance overrides if set, otherwise fall back to image defaults + // (like docker run overriding CMD) + entrypoint := imageInfo.Entrypoint + if len(inst.Entrypoint) > 0 { + entrypoint = inst.Entrypoint + } + cmd := imageInfo.Cmd + if len(inst.Cmd) > 0 { + cmd = inst.Cmd + } + cfg := &vmconfig.Config{ - Entrypoint: imageInfo.Entrypoint, - Cmd: imageInfo.Cmd, + Entrypoint: entrypoint, + Cmd: cmd, Workdir: imageInfo.WorkingDir, Env: mergeEnv(imageInfo.Env, inst.Env), InitMode: "exec", diff --git a/lib/instances/create.go b/lib/instances/create.go index 49550114..33724968 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -308,6 +308,8 @@ func (m *manager) createInstance( Devices: resolvedDeviceIDs, GPUProfile: gpuProfile, GPUMdevUUID: gpuMdevUUID, + Entrypoint: req.Entrypoint, + Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, SkipGuestAgent: req.SkipGuestAgent, } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index f1551045..6081332d 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -28,7 +28,7 @@ type Manager interface { StandbyInstance(ctx context.Context, id string) (*Instance, error) RestoreInstance(ctx context.Context, id string) (*Instance, error) StopInstance(ctx context.Context, id string) (*Instance, error) - StartInstance(ctx context.Context, id string) (*Instance, error) + StartInstance(ctx context.Context, id string, req StartInstanceRequest) (*Instance, error) StreamInstanceLogs(ctx context.Context, id string, tail int, follow bool, source LogSource) (<-chan string, error) RotateLogs(ctx context.Context, maxBytes int64, maxFiles int) error AttachVolume(ctx context.Context, id string, volumeId string, req AttachVolumeRequest) (*Instance, error) @@ -149,6 +149,15 @@ func (m *manager) getInstanceLock(id string) *sync.RWMutex { return lock.(*sync.RWMutex) } +// maybePersistExitInfo persists exit info to metadata under the instance write lock. +// Called from read paths when in-memory exit info was parsed but not yet persisted. +func (m *manager) maybePersistExitInfo(ctx context.Context, id string) { + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + m.persistExitInfo(ctx, id) +} + // CreateInstance creates and starts a new instance func (m *manager) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) { // Note: ID is generated inside createInstance, so we can't lock before calling it. @@ -197,12 +206,12 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error return m.stopInstance(ctx, id) } -// StartInstance starts a stopped instance -func (m *manager) StartInstance(ctx context.Context, id string) (*Instance, error) { +// StartInstance starts a stopped instance with optional command overrides +func (m *manager) StartInstance(ctx context.Context, id string, req StartInstanceRequest) (*Instance, error) { lock := m.getInstanceLock(id) lock.Lock() defer lock.Unlock() - return m.startInstance(ctx, id) + return m.startInstance(ctx, id, req) } // ListInstances returns all instances @@ -222,6 +231,11 @@ func (m *manager) GetInstance(ctx context.Context, idOrName string) (*Instance, inst, err := m.getInstance(ctx, idOrName) lock.RUnlock() if err == nil { + // If VM is stopped with unpersisted exit info, persist under write lock. + // This handles the "app exited on its own" case where stopInstance wasn't called. + if inst.State == StateStopped && inst.ExitCode != nil { + m.maybePersistExitInfo(ctx, inst.Id) + } return inst, nil } @@ -239,7 +253,11 @@ func (m *manager) GetInstance(ctx context.Context, idOrName string) (*Instance, } } if len(nameMatches) == 1 { - return &nameMatches[0], nil + inst := &nameMatches[0] + if inst.State == StateStopped && inst.ExitCode != nil { + m.maybePersistExitInfo(ctx, inst.Id) + } + return inst, nil } if len(nameMatches) > 1 { return nil, ErrAmbiguousName @@ -253,7 +271,11 @@ func (m *manager) GetInstance(ctx context.Context, idOrName string) (*Instance, } } if len(prefixMatches) == 1 { - return &prefixMatches[0], nil + inst := &prefixMatches[0] + if inst.State == StateStopped && inst.ExitCode != nil { + m.maybePersistExitInfo(ctx, inst.Id) + } + return inst, nil } if len(prefixMatches) > 1 { return nil, ErrAmbiguousName diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 7120903d..95c253bd 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -733,6 +733,39 @@ func TestBasicEndToEnd(t *testing.T) { } streamCancel() + // Test graceful stop: StopInstance sends Shutdown RPC -> init forwards SIGTERM + // -> app exits -> init writes exit sentinel -> reboot(POWER_OFF) -> VM stops cleanly + t.Log("Testing graceful stop via StopInstance...") + stoppedInst, err := manager.StopInstance(ctx, inst.Id) + require.NoError(t, err, "StopInstance should succeed") + assert.Equal(t, StateStopped, stoppedInst.State, "Instance should be in Stopped state after StopInstance") + + // Verify the instance reports Stopped on subsequent query and exit info is populated + retrieved, err = manager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Equal(t, StateStopped, retrieved.State, "Instance should remain Stopped") + require.NotNil(t, retrieved.ExitCode, "ExitCode should be populated after stop") + t.Logf("Exit code after graceful stop: %d, message: %q", *retrieved.ExitCode, retrieved.ExitMessage) + + t.Log("Graceful stop test passed!") + + // Test restart: StartInstance should clear stale exit info and boot the VM + t.Log("Testing restart after stop...") + restartedInst, err := manager.StartInstance(ctx, inst.Id, StartInstanceRequest{}) + require.NoError(t, err, "StartInstance should succeed") + assert.Equal(t, StateRunning, restartedInst.State, "Instance should be Running after restart") + + // Verify exit info was cleared + retrieved, err = manager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Nil(t, retrieved.ExitCode, "ExitCode should be nil after restart (stale exit info cleared)") + assert.Empty(t, retrieved.ExitMessage, "ExitMessage should be empty after restart") + t.Log("Restart test passed -- exit info cleared!") + + // Stop again before deleting + _, err = manager.StopInstance(ctx, inst.Id) + require.NoError(t, err) + // Delete instance t.Log("Deleting instance...") err = manager.DeleteInstance(ctx, inst.Id) @@ -763,6 +796,182 @@ func TestBasicEndToEnd(t *testing.T) { t.Log("Instance and volume lifecycle test complete!") } +// TestAppExitPropagation verifies the full exit info pipeline when an app exits on its own: +// app exits -> init writes HYPEMAN-EXIT sentinel -> reboot(POWER_OFF) -> VM stops -> +// host lazily parses sentinel from serial log -> ExitCode/ExitMessage in metadata. +// Uses alpine with a non-existent command override to get exit code 127 ("command not found"). +func TestAppExitPropagation(t *testing.T) { + if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { + t.Skip("/dev/kvm not available, skipping on this platform") + } + + manager, tmpDir := setupTestManager(t) + ctx := context.Background() + p := paths.New(tmpDir) + + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + + t.Log("Pulling alpine:latest image...") + alpineImage, err := imageManager.CreateImage(ctx, images.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + }) + require.NoError(t, err) + + // Wait for image to be ready + imageName := alpineImage.Name + for i := 0; i < 60; i++ { + img, err := imageManager.GetImage(ctx, imageName) + if err == nil && img.Status == images.StatusReady { + alpineImage = img + break + } + if err == nil && img.Status == images.StatusFailed { + t.Fatalf("Image build failed: %s", *img.Error) + } + time.Sleep(1 * time.Second) + } + require.Equal(t, images.StatusReady, alpineImage.Status) + t.Log("Alpine image ready") + + // Ensure system files + systemManager := system.NewManager(p) + err = systemManager.EnsureSystemFiles(ctx) + require.NoError(t, err) + + // Create instance with a non-existent command (like `docker run alpine /nonexistent`). + // This overrides alpine's default CMD ("/bin/sh") with a command that doesn't exist, + // causing exit code 127 ("command not found"). + inst, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "test-exit-propagation", + Image: "docker.io/library/alpine:latest", + Size: 512 * 1024 * 1024, // 512MB + HotplugSize: 0, + OverlaySize: 2 * 1024 * 1024 * 1024, // 2GB + Vcpus: 1, + Cmd: []string{"/nonexistent-command"}, + }) + require.NoError(t, err) + t.Logf("Instance created: %s", inst.Id) + + // Wait for VM to reach running state first + err = waitForVMReady(ctx, inst.SocketPath, 10*time.Second) + require.NoError(t, err, "VM should reach running state") + + // Wait for the VM to stop on its own (/nonexistent-command exits 127 immediately). + // Poll GetInstance until state becomes Stopped (init writes sentinel then reboots). + t.Log("Waiting for VM to stop on its own (expecting exit 127)...") + var finalInst *Instance + for i := 0; i < 60; i++ { // up to 60 seconds + got, err := manager.GetInstance(ctx, inst.Id) + if err == nil && got.State == StateStopped { + finalInst = got + break + } + time.Sleep(1 * time.Second) + } + require.NotNil(t, finalInst, "Instance should reach Stopped state within 60 seconds") + assert.Equal(t, StateStopped, finalInst.State) + + // Verify exit info was propagated from the serial console sentinel + require.NotNil(t, finalInst.ExitCode, "ExitCode should be populated after app exits") + assert.Equal(t, 127, *finalInst.ExitCode, "Non-existent command should exit with code 127") + assert.Contains(t, finalInst.ExitMessage, "command not found", "Exit message should say command not found") + t.Logf("Exit info propagated: code=%d message=%q", *finalInst.ExitCode, finalInst.ExitMessage) + + // Cleanup + err = manager.DeleteInstance(ctx, inst.Id) + require.NoError(t, err) + + t.Log("App exit propagation test complete!") +} + +// TestOOMExitPropagation verifies that OOM kills are detected and reported. +// Creates a VM with low memory and runs a command that allocates more than available, +// triggering the OOM killer. Verifies exit code 137 and "OOM" in the exit message. +func TestOOMExitPropagation(t *testing.T) { + if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { + t.Skip("/dev/kvm not available, skipping on this platform") + } + + manager, tmpDir := setupTestManager(t) + ctx := context.Background() + p := paths.New(tmpDir) + + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + + t.Log("Pulling alpine:latest image...") + alpineImage, err := imageManager.CreateImage(ctx, images.CreateImageRequest{ + Name: "docker.io/library/alpine:latest", + }) + require.NoError(t, err) + + imageName := alpineImage.Name + for i := 0; i < 60; i++ { + img, err := imageManager.GetImage(ctx, imageName) + if err == nil && img.Status == images.StatusReady { + alpineImage = img + break + } + if err == nil && img.Status == images.StatusFailed { + t.Fatalf("Image build failed: %s", *img.Error) + } + time.Sleep(1 * time.Second) + } + require.Equal(t, images.StatusReady, alpineImage.Status) + t.Log("Alpine image ready") + + systemManager := system.NewManager(p) + err = systemManager.EnsureSystemFiles(ctx) + require.NoError(t, err) + + // Create instance with minimal memory (256MB) and a command that allocates + // anonymous memory until the OOM killer fires and kills the process with SIGKILL. + // We use a shell script that creates a large string variable in a loop, forcing + // the shell process to grow its RSS until OOM kills it. + inst, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "test-oom", + Image: "docker.io/library/alpine:latest", + Size: 128 * 1024 * 1024, // 128MB -- small enough for OOM + HotplugSize: 0, + OverlaySize: 2 * 1024 * 1024 * 1024, // 2GB + Vcpus: 1, + Cmd: []string{"sh", "-c", "a=x; while true; do a=$a$a$a$a; done"}, + }) + require.NoError(t, err) + t.Logf("Instance created: %s (128MB RAM, will OOM)", inst.Id) + + err = waitForVMReady(ctx, inst.SocketPath, 10*time.Second) + require.NoError(t, err, "VM should reach running state") + + // Wait for the VM to stop (OOM kill -> init detects -> sentinel -> reboot) + t.Log("Waiting for VM to stop after OOM...") + var finalInst *Instance + for i := 0; i < 90; i++ { // up to 90 seconds (OOM may take time with low memory) + got, err := manager.GetInstance(ctx, inst.Id) + if err == nil && got.State == StateStopped { + finalInst = got + break + } + time.Sleep(1 * time.Second) + } + require.NotNil(t, finalInst, "Instance should reach Stopped state within 90 seconds") + assert.Equal(t, StateStopped, finalInst.State) + + // Verify exit info shows OOM + require.NotNil(t, finalInst.ExitCode, "ExitCode should be populated after OOM") + assert.Equal(t, 137, *finalInst.ExitCode, "OOM kill should result in exit code 137 (SIGKILL)") + assert.Contains(t, finalInst.ExitMessage, "OOM", "Exit message should indicate OOM") + t.Logf("OOM exit info propagated: code=%d message=%q", *finalInst.ExitCode, finalInst.ExitMessage) + + // Cleanup + err = manager.DeleteInstance(ctx, inst.Id) + require.NoError(t, err) + + t.Log("OOM exit propagation test complete!") +} + // TestEntrypointEnvVars verifies that environment variables are passed to the entrypoint process. // This uses bitnami/redis which configures REDIS_PASSWORD from an env var - if auth is required, // it proves the entrypoint received and used the env var. diff --git a/lib/instances/qemu_test.go b/lib/instances/qemu_test.go index 98d0095e..f2ffef4f 100644 --- a/lib/instances/qemu_test.go +++ b/lib/instances/qemu_test.go @@ -523,6 +523,22 @@ func TestQEMUBasicEndToEnd(t *testing.T) { assert.Equal(t, "test_value", strings.TrimSpace(output), "Environment variable should be accessible via exec") t.Log("Environment variable accessible via exec!") + // Test graceful stop: StopInstance sends Shutdown RPC -> init forwards SIGTERM + // -> app exits -> init writes exit sentinel -> reboot(POWER_OFF) -> VM stops cleanly + t.Log("Testing graceful stop via StopInstance...") + stoppedInst, err := manager.StopInstance(ctx, inst.Id) + require.NoError(t, err, "StopInstance should succeed") + assert.Equal(t, StateStopped, stoppedInst.State, "Instance should be in Stopped state after StopInstance") + + // Verify the instance reports Stopped on subsequent query and exit info is populated + retrieved, err = manager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Equal(t, StateStopped, retrieved.State, "Instance should remain Stopped") + require.NotNil(t, retrieved.ExitCode, "ExitCode should be populated after stop") + t.Logf("Exit code after graceful stop: %d, message: %q", *retrieved.ExitCode, retrieved.ExitMessage) + + t.Log("Graceful stop test passed!") + // Delete instance t.Log("Deleting instance...") err = manager.DeleteInstance(ctx, inst.Id) diff --git a/lib/instances/query.go b/lib/instances/query.go index 1bc26fc2..49d89244 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -3,13 +3,19 @@ package instances import ( "context" "fmt" + "io" "os" "path/filepath" + "strconv" + "strings" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" ) +// exitSentinelPrefix is the machine-parseable prefix written by init to serial console. +const exitSentinelPrefix = "HYPEMAN-EXIT " + // stateResult holds the result of state derivation type stateResult struct { State State @@ -104,9 +110,157 @@ func (m *manager) toInstance(ctx context.Context, meta *metadata) Instance { StateError: result.Error, HasSnapshot: m.hasSnapshot(meta.StoredMetadata.DataDir), } + + // If VM is stopped and exit info isn't persisted yet, populate in-memory + // from the serial console log. This is read-only -- no metadata writes. + // Persistence happens under lock in stopInstance or persistExitInfo. + if inst.State == StateStopped && inst.ExitCode == nil { + if code, msg, ok := m.parseExitSentinel(inst.Id); ok { + inst.ExitCode = &code + inst.ExitMessage = msg + } + } + return inst } +// parseExitSentinel reads the last lines of the serial console log to find the +// HYPEMAN-EXIT sentinel written by init before shutdown. +// Returns the exit code, message, and whether a sentinel was found. +// This is a pure reader with no side effects. +func (m *manager) parseExitSentinel(id string) (int, string, bool) { + logPath := m.paths.InstanceAppLog(id) + + // Read the tail of the log file. The sentinel is written near the end + // (just before reboot), so we only need the last few KB even if the + // serial console log is large from a chatty app. + const tailSize = 8192 + data, err := readTail(logPath, tailSize) + if err != nil { + return 0, "", false + } + + // Scan lines from the tail looking for the sentinel + lines := strings.Split(string(data), "\n") + for _, line := range lines { + code, msg, ok := parseExitSentinelLine(line) + if ok { + return code, msg, true + } + } + return 0, "", false +} + +// persistExitInfo parses exit info from the serial console and persists it to +// metadata. Must be called under the instance lock. +func (m *manager) persistExitInfo(ctx context.Context, id string) { + log := logger.FromContext(ctx) + + meta, err := m.loadMetadata(id) + if err != nil { + return + } + + // Already persisted + if meta.ExitCode != nil { + return + } + + code, msg, ok := m.parseExitSentinel(id) + if !ok { + return + } + + meta.ExitCode = &code + meta.ExitMessage = msg + if err := m.saveMetadata(meta); err != nil { + log.WarnContext(ctx, "failed to persist exit info", "instance_id", id, "error", err) + } else { + log.DebugContext(ctx, "parsed exit info from serial log", "instance_id", id, "exit_code", code, "exit_message", msg) + } +} + +// readTail reads the last n bytes of a file. If the file is smaller than n, +// the entire file is returned. +func readTail(path string, n int64) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return nil, err + } + + offset := info.Size() - n + if offset < 0 { + offset = 0 + } + + if offset > 0 { + if _, err := f.Seek(offset, io.SeekStart); err != nil { + return nil, err + } + } + + return io.ReadAll(f) +} + +// parseExitSentinelLine parses a single log line looking for the HYPEMAN-EXIT sentinel. +// The sentinel format is embedded in a log line like: +// 2026-02-13T15:26:27Z [INFO] [hypeman-init:entrypoint] HYPEMAN-EXIT code=127 message="command not found" +// Returns the exit code, message, and whether parsing was successful. +func parseExitSentinelLine(line string) (int, string, bool) { + // Strip whitespace -- serial console (TTY) adds \r to line endings + line = strings.TrimSpace(line) + + idx := strings.Index(line, exitSentinelPrefix) + if idx < 0 { + return 0, "", false + } + + // Extract the part after "HYPEMAN-EXIT " + sentinel := line[idx+len(exitSentinelPrefix):] + + // Parse code=N + if !strings.HasPrefix(sentinel, "code=") { + return 0, "", false + } + sentinel = sentinel[5:] // skip "code=" + + // Find the end of the code number + spaceIdx := strings.Index(sentinel, " ") + if spaceIdx < 0 { + // Just a code, no message + code, err := strconv.Atoi(sentinel) + if err != nil { + return 0, "", false + } + return code, "", true + } + + code, err := strconv.Atoi(sentinel[:spaceIdx]) + if err != nil { + return 0, "", false + } + + // Parse message="..." + rest := sentinel[spaceIdx+1:] + if strings.HasPrefix(rest, "message=") { + msgStr := rest[8:] // skip "message=" + // Unquote the message (it's Go-quoted via %q) + if unquoted, err := strconv.Unquote(msgStr); err == nil { + return code, unquoted, true + } + // If unquoting fails, use raw value (strip quotes if present) + return code, strings.Trim(msgStr, "\""), true + } + + return code, "", true +} + // listInstances returns all instances func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { log := logger.FromContext(ctx) diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go new file mode 100644 index 00000000..b3cd0873 --- /dev/null +++ b/lib/instances/query_test.go @@ -0,0 +1,92 @@ +package instances + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseExitSentinelLine(t *testing.T) { + tests := []struct { + name string + line string + wantOK bool + wantCode int + wantMsg string + }{ + { + name: "standard log line with sentinel", + line: `2026-02-13T15:26:27Z [INFO] [hypeman-init:entrypoint] HYPEMAN-EXIT code=127 message="command not found"`, + wantOK: true, + wantCode: 127, + wantMsg: "command not found", + }, + { + name: "exit code 0", + line: `2026-02-13T15:26:27Z [INFO] [hypeman-init:entrypoint] HYPEMAN-EXIT code=0 message="success"`, + wantOK: true, + wantCode: 0, + wantMsg: "success", + }, + { + name: "SIGKILL with OOM", + line: `2026-02-13T15:26:27Z [INFO] [hypeman-init:entrypoint] HYPEMAN-EXIT code=137 message="killed by signal 9 (killed) - OOM"`, + wantOK: true, + wantCode: 137, + wantMsg: "killed by signal 9 (killed) - OOM", + }, + { + name: "message with escaped quotes", + line: `HYPEMAN-EXIT code=1 message="error: \"bad thing\""`, + wantOK: true, + wantCode: 1, + wantMsg: `error: "bad thing"`, + }, + { + name: "no sentinel", + line: "2026-02-13T15:26:27Z [INFO] [hypeman-init:entrypoint] app exited with code 127", + wantOK: false, + }, + { + name: "empty line", + line: "", + wantOK: false, + }, + { + name: "partial sentinel", + line: "HYPEMAN-EXIT", + wantOK: false, + }, + { + name: "sentinel without message", + line: "HYPEMAN-EXIT code=42", + wantOK: true, + wantCode: 42, + wantMsg: "", + }, + { + name: "invalid code", + line: "HYPEMAN-EXIT code=abc message=\"error\"", + wantOK: false, + }, + { + name: "line with carriage return from serial console", + line: "2026-02-13T15:26:27Z [INFO] [hypeman-init:entrypoint] HYPEMAN-EXIT code=0 message=\"success\"\r", + wantOK: true, + wantCode: 0, + wantMsg: "success", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + code, msg, ok := parseExitSentinelLine(tc.line) + require.Equal(t, tc.wantOK, ok, "parseExitSentinelLine(%q) ok=%v, want %v", tc.line, ok, tc.wantOK) + if ok { + assert.Equal(t, tc.wantCode, code, "exit code mismatch") + assert.Equal(t, tc.wantMsg, msg, "exit message mismatch") + } + }) + } +} diff --git a/lib/instances/start.go b/lib/instances/start.go index 9eaf60ed..2e257aeb 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -17,6 +17,7 @@ import ( func (m *manager) startInstance( ctx context.Context, id string, + req StartInstanceRequest, ) (*Instance, error) { start := time.Now() log := logger.FromContext(ctx) @@ -46,6 +47,16 @@ func (m *manager) startInstance( return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + // 2a. Clear stale exit info from previous run and apply command overrides + stored.ExitCode = nil + stored.ExitMessage = "" + if len(req.Entrypoint) > 0 { + stored.Entrypoint = req.Entrypoint + } + if len(req.Cmd) > 0 { + stored.Cmd = req.Cmd + } + // 2b. Validate aggregate resource limits before allocating resources (if configured) if m.resourceValidator != nil { needsGPU := stored.GPUProfile != "" diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 973f7641..fafc65c0 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -6,12 +6,19 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" "go.opentelemetry.io/otel/trace" ) -// stopInstance gracefully stops a running instance +// DefaultStopTimeout is the default grace period for graceful shutdown (seconds). +// Similar to Docker's default of 10s. +const DefaultStopTimeout = 10 + +// stopInstance gracefully stops a running instance. +// Flow: send Shutdown RPC -> wait for VM to power off -> fall back to hard kill. // Multi-hop orchestration: Running → Shutdown → Stopped func (m *manager) stopInstance( ctx context.Context, @@ -55,15 +62,49 @@ func (m *manager) stopInstance( } } - // 4. Shutdown hypervisor process - // TODO: Add graceful shutdown via vsock signal to allow app to clean up - log.DebugContext(ctx, "shutting down hypervisor", "instance_id", id) - if err := m.shutdownHypervisor(ctx, &inst); err != nil { - // Log but continue - try to clean up anyway - log.WarnContext(ctx, "failed to shutdown hypervisor gracefully", "instance_id", id, "error", err) + // 4. Graceful shutdown: send signal to guest init via Shutdown RPC, + // then wait for VM to power off cleanly. Fall back to hard kill on timeout. + stopTimeout := stored.StopTimeout + if stopTimeout <= 0 { + stopTimeout = DefaultStopTimeout + } + + gracefulShutdown := false + if !stored.SkipGuestAgent { + log.DebugContext(ctx, "sending graceful shutdown signal to guest", "instance_id", id, "timeout_seconds", stopTimeout) + dialer, dialerErr := hypervisor.NewVsockDialer(stored.HypervisorType, stored.VsockSocket, stored.VsockCID) + if dialerErr != nil { + log.WarnContext(ctx, "could not create vsock dialer for graceful shutdown", "instance_id", id, "error", dialerErr) + } else { + // Send shutdown signal (best-effort, fire and forget) + shutdownCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + if err := guest.ShutdownInstance(shutdownCtx, dialer, 0); err != nil { + log.WarnContext(ctx, "shutdown RPC failed (will hard kill)", "instance_id", id, "error", err) + } + cancel() + + // Wait for the hypervisor process to exit (init calls reboot(POWER_OFF)) + if inst.HypervisorPID != nil { + if WaitForProcessExit(*inst.HypervisorPID, time.Duration(stopTimeout)*time.Second) { + log.DebugContext(ctx, "VM shut down gracefully", "instance_id", id) + gracefulShutdown = true + } else { + log.WarnContext(ctx, "graceful shutdown timed out, falling back to hard kill", "instance_id", id) + } + } + } } - // 5. Release network allocation (delete TAP device) + // 5. Hard kill if graceful shutdown didn't work + if !gracefulShutdown { + log.DebugContext(ctx, "shutting down hypervisor (hard kill)", "instance_id", id) + if err := m.shutdownHypervisor(ctx, &inst); err != nil { + // Log but continue - try to clean up anyway + log.WarnContext(ctx, "failed to shutdown hypervisor", "instance_id", id, "error", err) + } + } + + // 6. Release network allocation (delete TAP device) if inst.NetworkEnabled && networkAlloc != nil { log.DebugContext(ctx, "releasing network", "instance_id", id, "network", "default") if err := m.networkManager.ReleaseAllocation(ctx, networkAlloc); err != nil { @@ -72,7 +113,7 @@ func (m *manager) stopInstance( } } - // 6. Destroy vGPU mdev device if present (frees vGPU slot for other VMs) + // 7. Destroy vGPU mdev device if present (frees vGPU slot for other VMs) if inst.GPUMdevUUID != "" { log.InfoContext(ctx, "destroying vGPU mdev on stop", "instance_id", id, "uuid", inst.GPUMdevUUID) if err := devices.DestroyMdev(ctx, inst.GPUMdevUUID); err != nil { @@ -81,7 +122,7 @@ func (m *manager) stopInstance( } } - // 7. Update metadata (clear PID, mdev UUID, set StoppedAt) + // 8. Update metadata (clear PID, mdev UUID, set StoppedAt) now := time.Now() stored.StoppedAt = &now stored.HypervisorPID = nil @@ -93,6 +134,9 @@ func (m *manager) stopInstance( return nil, fmt.Errorf("save metadata: %w", err) } + // 9. Persist exit info from serial console (under lock, safe from races) + m.persistExitInfo(ctx, id) + // Record metrics if m.metrics != nil { m.recordDuration(ctx, m.metrics.stopDuration, start, "success", stored.HypervisorType) diff --git a/lib/instances/types.go b/lib/instances/types.go index 770abcd2..64966c9a 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -82,9 +82,20 @@ type StoredMetadata struct { GPUProfile string // vGPU profile name (e.g., "L40S-1Q") GPUMdevUUID string // mdev device UUID + // Command overrides (like docker run ) + Entrypoint []string // Override image entrypoint (nil = use image default) + Cmd []string // Override image cmd (nil = use image default) + // Boot optimizations SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) + + // Shutdown configuration + StopTimeout int // Grace period in seconds for graceful stop (0 = use default 10s) + + // Exit information (populated from serial console sentinel when VM stops) + ExitCode *int // App exit code, nil if VM hasn't exited + ExitMessage string // Human-readable description of exit (e.g., "command not found", "killed by signal 9 (SIGKILL) - OOM") } // Instance represents a virtual machine instance with derived runtime state @@ -126,10 +137,18 @@ type CreateInstanceRequest struct { Volumes []VolumeAttachment // Volumes to attach at creation time Hypervisor hypervisor.Type // Optional: hypervisor type (defaults to config) GPU *GPUConfig // Optional: vGPU configuration + Entrypoint []string // Override image entrypoint (nil = use image default) + Cmd []string // Override image cmd (nil = use image default) SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) } +// StartInstanceRequest is the domain request for starting a stopped instance +type StartInstanceRequest struct { + Entrypoint []string // Override entrypoint (nil = keep previous/image default) + Cmd []string // Override cmd (nil = keep previous/image default) +} + // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) type AttachVolumeRequest struct { MountPath string diff --git a/lib/instances/volumes_test.go b/lib/instances/volumes_test.go index d1614f8d..79a886b8 100644 --- a/lib/instances/volumes_test.go +++ b/lib/instances/volumes_test.go @@ -97,6 +97,7 @@ func TestVolumeMultiAttachReadOnly(t *testing.T) { HotplugSize: 512 * 1024 * 1024, OverlaySize: 1024 * 1024 * 1024, Vcpus: 1, + Cmd: []string{"sleep", "infinity"}, // Keep VM alive for exec NetworkEnabled: false, Volumes: []VolumeAttachment{ {VolumeID: vol.Id, MountPath: "/data", Readonly: false}, @@ -139,6 +140,7 @@ func TestVolumeMultiAttachReadOnly(t *testing.T) { HotplugSize: 512 * 1024 * 1024, OverlaySize: 1024 * 1024 * 1024, Vcpus: 1, + Cmd: []string{"sleep", "infinity"}, // Keep VM alive for exec NetworkEnabled: false, Volumes: []VolumeAttachment{ {VolumeID: vol.Id, MountPath: "/data", Readonly: true}, @@ -155,6 +157,7 @@ func TestVolumeMultiAttachReadOnly(t *testing.T) { HotplugSize: 512 * 1024 * 1024, OverlaySize: 1024 * 1024 * 1024, Vcpus: 1, + Cmd: []string{"sleep", "infinity"}, // Keep VM alive for exec NetworkEnabled: false, Volumes: []VolumeAttachment{ {VolumeID: vol.Id, MountPath: "/data", Readonly: true, Overlay: true, OverlaySize: 100 * 1024 * 1024}, // 100MB overlay @@ -274,6 +277,7 @@ func TestOverlayDiskCleanupOnDelete(t *testing.T) { HotplugSize: 512 * 1024 * 1024, OverlaySize: 1024 * 1024 * 1024, Vcpus: 1, + Cmd: []string{"sleep", "infinity"}, // Keep VM alive for exec NetworkEnabled: false, Volumes: []VolumeAttachment{ {VolumeID: vol.Id, MountPath: "/data", Readonly: true, Overlay: true, OverlaySize: 100 * 1024 * 1024}, @@ -398,6 +402,7 @@ func TestVolumeFromArchive(t *testing.T) { HotplugSize: 512 * 1024 * 1024, OverlaySize: 1024 * 1024 * 1024, Vcpus: 1, + Cmd: []string{"sleep", "infinity"}, // Keep VM alive for exec NetworkEnabled: false, Volumes: []VolumeAttachment{ {VolumeID: vol.Id, MountPath: "/archive", Readonly: true}, diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index ebd3e42d..a389712e 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -239,12 +239,18 @@ type CreateIngressRequest struct { // CreateInstanceRequest defines model for CreateInstanceRequest. type CreateInstanceRequest struct { + // Cmd Override image CMD (like docker run ). Omit to use image default. + Cmd *[]string `json:"cmd,omitempty"` + // Devices Device IDs or names to attach for GPU/PCI passthrough Devices *[]string `json:"devices,omitempty"` // DiskIoBps Disk I/O rate limit (e.g., "100MB/s", "500MB/s"). Defaults to proportional share based on CPU allocation if configured. DiskIoBps *string `json:"disk_io_bps,omitempty"` + // Entrypoint Override image entrypoint (like docker run --entrypoint). Omit to use image default. + Entrypoint *[]string `json:"entrypoint,omitempty"` + // Env Environment variables Env *map[string]string `json:"env,omitempty"` @@ -260,7 +266,7 @@ type CreateInstanceRequest struct { // Image OCI image reference Image string `json:"image"` - // Metadata User-defined key-value metadata + // Metadata User-defined key-value metadata for the instance Metadata *map[string]string `json:"metadata,omitempty"` // Name Human-readable name (lowercase letters, digits, and dashes only; cannot start or end with a dash) @@ -544,6 +550,12 @@ type Instance struct { // Env Environment variables Env *map[string]string `json:"env,omitempty"` + // ExitCode App exit code (null if VM hasn't exited) + ExitCode *int `json:"exit_code"` + + // ExitMessage Human-readable description of exit (e.g., "command not found", "killed by signal 9 (SIGKILL) - OOM") + ExitMessage *string `json:"exit_message,omitempty"` + // Gpu GPU information attached to the instance Gpu *InstanceGPU `json:"gpu,omitempty"` @@ -881,6 +893,15 @@ type GetInstanceLogsParams struct { // GetInstanceLogsParamsSource defines parameters for GetInstanceLogs. type GetInstanceLogsParamsSource string +// StartInstanceJSONBody defines parameters for StartInstance. +type StartInstanceJSONBody struct { + // Cmd Override image CMD for this run. Omit to keep previous value. + Cmd *[]string `json:"cmd,omitempty"` + + // Entrypoint Override image entrypoint for this run. Omit to keep previous value. + Entrypoint *[]string `json:"entrypoint,omitempty"` +} + // StatInstancePathParams defines parameters for StatInstancePath. type StatInstancePathParams struct { // Path Path to stat in the guest filesystem @@ -917,6 +938,9 @@ type CreateIngressJSONRequestBody = CreateIngressRequest // CreateInstanceJSONRequestBody defines body for CreateInstance for application/json ContentType. type CreateInstanceJSONRequestBody = CreateInstanceRequest +// StartInstanceJSONRequestBody defines body for StartInstance for application/json ContentType. +type StartInstanceJSONRequestBody StartInstanceJSONBody + // AttachVolumeJSONRequestBody defines body for AttachVolume for application/json ContentType. type AttachVolumeJSONRequestBody = AttachVolumeRequest @@ -1082,8 +1106,10 @@ type ClientInterface interface { // StandbyInstance request StandbyInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) - // StartInstance request - StartInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // StartInstanceWithBody request with any body + StartInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + StartInstance(ctx context.Context, id string, body StartInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) // StatInstancePath request StatInstancePath(ctx context.Context, id string, params *StatInstancePathParams, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -1483,8 +1509,20 @@ func (c *Client) StandbyInstance(ctx context.Context, id string, reqEditors ...R return c.Client.Do(req) } -func (c *Client) StartInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewStartInstanceRequest(c.Server, id) +func (c *Client) StartInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewStartInstanceRequestWithBody(c.Server, id, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) StartInstance(ctx context.Context, id string, body StartInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewStartInstanceRequest(c.Server, id, body) if err != nil { return nil, err } @@ -2581,8 +2619,19 @@ func NewStandbyInstanceRequest(server string, id string) (*http.Request, error) return req, nil } -// NewStartInstanceRequest generates requests for StartInstance -func NewStartInstanceRequest(server string, id string) (*http.Request, error) { +// NewStartInstanceRequest calls the generic StartInstance builder with application/json body +func NewStartInstanceRequest(server string, id string, body StartInstanceJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewStartInstanceRequestWithBody(server, id, "application/json", bodyReader) +} + +// NewStartInstanceRequestWithBody generates requests for StartInstance with any type of body +func NewStartInstanceRequestWithBody(server string, id string, contentType string, body io.Reader) (*http.Request, error) { var err error var pathParam0 string @@ -2607,11 +2656,13 @@ func NewStartInstanceRequest(server string, id string) (*http.Request, error) { return nil, err } - req, err := http.NewRequest("POST", queryURL.String(), nil) + req, err := http.NewRequest("POST", queryURL.String(), body) if err != nil { return nil, err } + req.Header.Add("Content-Type", contentType) + return req, nil } @@ -3212,8 +3263,10 @@ type ClientWithResponsesInterface interface { // StandbyInstanceWithResponse request StandbyInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) - // StartInstanceWithResponse request - StartInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StartInstanceResponse, error) + // StartInstanceWithBodyWithResponse request with any body + StartInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StartInstanceResponse, error) + + StartInstanceWithResponse(ctx context.Context, id string, body StartInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*StartInstanceResponse, error) // StatInstancePathWithResponse request StatInstancePathWithResponse(ctx context.Context, id string, params *StatInstancePathParams, reqEditors ...RequestEditorFn) (*StatInstancePathResponse, error) @@ -4445,9 +4498,17 @@ func (c *ClientWithResponses) StandbyInstanceWithResponse(ctx context.Context, i return ParseStandbyInstanceResponse(rsp) } -// StartInstanceWithResponse request returning *StartInstanceResponse -func (c *ClientWithResponses) StartInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StartInstanceResponse, error) { - rsp, err := c.StartInstance(ctx, id, reqEditors...) +// StartInstanceWithBodyWithResponse request with arbitrary body returning *StartInstanceResponse +func (c *ClientWithResponses) StartInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StartInstanceResponse, error) { + rsp, err := c.StartInstanceWithBody(ctx, id, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseStartInstanceResponse(rsp) +} + +func (c *ClientWithResponses) StartInstanceWithResponse(ctx context.Context, id string, body StartInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*StartInstanceResponse, error) { + rsp, err := c.StartInstance(ctx, id, body, reqEditors...) if err != nil { return nil, err } @@ -8945,7 +9006,8 @@ func (response StandbyInstance500JSONResponse) VisitStandbyInstanceResponse(w ht } type StartInstanceRequestObject struct { - Id string `json:"id"` + Id string `json:"id"` + Body *StartInstanceJSONRequestBody } type StartInstanceResponseObject interface { @@ -10280,6 +10342,13 @@ func (sh *strictHandler) StartInstance(w http.ResponseWriter, r *http.Request, i request.Id = id + var body StartInstanceJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { return sh.ssi.StartInstance(ctx, request.(StartInstanceRequestObject)) } @@ -10602,168 +10671,173 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9a3PburXoX8Hwnk7lU0mWH3Ecn+nccewk222c+Maxe0+3chWIhCRskwADgHKUTL72", - "B/Qn7l9yBwsAXwIlKg8nPjudzg4tgngsrDcW1voYhDxJOSNMyeDoYyDDGUkwPB4rhcPZNY+zhLwi7zIi", - "lf45FTwlQlECjRKeMTVKsZrpvyIiQ0FTRTkLjoILrGbodkYEQXPoBckZz+IIjQmC70gUdAPyHidpTIKj", - "YDthajvCCgfdQC1S/ZNUgrJp8KkbCIIjzuKFGWaCs1gFRxMcS9KtDXuuu0ZYIv1JD77J+xtzHhPMgk/Q", - "47uMChIFR7+Wl/Emb8zHv5FQ6cGP55jGeByTUzKnIVkGQ5gJQZgaRYLOiVgGxYl5Hy/QmGcsQqYd6rAs", - "jhGdIMYZ2aoAg81pRDUkdBM9dHCkREY8kIlgTiMaeXbg5AyZ1+jsFHVm5H11kN2H48OguUuGE7Lc6S9Z", - "gllPA1dPy/UPbct9P9/39Ux5kmSjqeBZutzz2cvz8ysELxHLkjER5R4Pd/P+KFNkSoTuMA3pCEeRIFL6", - "1+9eluc2GAwGR3j3aDDoD3yznBMWcdEIUvPaD9KdQURWdNkKpLb/JZC+uD47PTtGJ1ykXGD4dmmkGmKX", - "wVNeVxltqrviw//HGY2jZawf65+JGFEmFWYNOHhmX2pw8QlSM4Lsd+j6HHUmXKCIjLPplLLpVht81wwr", - "JopEI6yWh4OpItuGcoYUTYhUOEmDbjDhItEfBRFWpKfftBpQELxmON2i1WDLpJaZnRwlsql31wRRhhIa", - "x1SSkLNIlsegTB3sNy+mRDBECO7hUE/0zyghUuIpQR3NNjXvZkgqrDKJqEQTTGMStdojHyKYxfzGx4hG", - "hCk6oVX6NujUw+NwZ3fPyzsSPCWjiE6tJKp2fwq/axTT/SgErf0L0YS2aLcOGFKQyfJ4T4F1wyCCTIgg", - "Gse/cLhU8Dlhmlr0eP8B4wb/a7sQ0dtWPm8DMC+K5p+6wbuMZGSUcknNDJc4l32j0QhAjeAL/5zh1aq9", - "LmGUVFispg9o8RUo0cyvFWwuTdM6PwR2Z7upUHYj23syJ8yj+IScKfuiuuLnfIpiygiyLSx8NZ/TA/w1", - "5sDmvsbaukEB0mWC1vP+DIZkfmjoTb/rBoRliQZmzKdlaM4IFmpMKsBsEEu2o2J2jeC/qJBETf5gSUar", - "ucIFZYxESLe0xGpaokyC9rm0fKCMG6pGcyKkl45gWn+nCtkWjV3FPLyZ0JiMZljOzIxxFAEN4viishKP", - "BlZRaXGqGZvrEDQDiRRHl78c7z44QHYADwwlz0RoZrC8ktLXunvTFiksxjiOvbjRjG6by91lDPFjwGVO", - "GE3yJMdAh5iGewV2N3X33SDN5Mw8AT/WswJ5ptmARq9YP7/xLPoEmITR/BvtIL9e9zI1m42mMdcwXaCM", - "0XdZRWnuozOt/yukmT+NSNRFGF5oNowzxXtTwojQfApNBE9AgyoptqhD+tN+Fw21rtfTmm0P7/YGg95g", - "GFRV03i/N00zDQqsFBF6gv/vV9z7cNz756D36E3xOOr33vzlP3wI0FbbdpqeXWfH0X4XucmWVfD6RFer", - "5ys0XB8XMdt3pml/0907OVsW8Gb+EQ9viOhTvh3TscBisc2mlL0/irEiUlVXs7rt2vXB3FYsjE310jdc", - "Ws3gAHTrxPyWiFBzyphoBJFdzSypkl2Etc0KTAZpafZfKMRM46wR7FwgwiJ0S9UMYWhXhUCy6OGU9qiZ", - "atANEvz+OWFTNQuODvaW8FEjY8c+9N78p/tp6397UVJkMfEg4yueKcqmCF4b6TujEhVzoIoka8Wtg24W", - "g4qVUHZmPtvJZ4KFwAv/rrnJrdo9Yxw1bp8hIM/6Tp1ZL5E1FUEgYHDawHqfXVxta5JMsZRqJng2nZV3", - "5VfHD96UYNGgDbhFdoOIypsR5aNx6psTlTfobPsl0twKxTShquBOO4PB+eNtOQz0Hw/cH1t9dGq8OTB9", - "vXguLNOUMywIiO4IcYZOLq4QjmMeWmNoojWsCZ1mgkT9mg0OvfuwhbD5F8jhJ2xOBWeJ1oXmWFBNPBXP", - "wsfgxcvTJ6MnL66DI72TURZaM/3i5avXwVGwNxgMAp+o0zuxBhmfXVydwIp1+xlXaZxNR5J+IBWfWLD3", - "7HFQn/hxvl6UkIQLo4/aPlBnVmUHRlyjmN4QNNT9mU3beVZn1Lsw1BLQZouUiDmVPjvzl/yd3u9MkjJt", - "GmKoooQkYk5Evtew+f2SrA9jnkW90pDd4B1JtJibf9C4XczW09Jv8LUSBWt4PI5Tykgjk+/+KIz5loub", - "mOOot/OV+TIjSve9vMQX5kV1Ry0WkBwJgu6Sss+iWxqp2Sjit0xP2cOA7BuUN8650Hu9Ehz//q9/X58X", - "WsjOs3FqWdLO7oMvZEk1JqS79loY+UKy1L+Mq9S/iOvz3//1b7eS77sIwjR+RhXOY4z26lL+MSNqRkRJ", - "NLkN1j8ZFRE+Rw5fSsNXvABl1/0S9+RzImK88HDDnYGHHf5DUAX0Zb9DWqwh/fEaXqh7cxJsmRsO/OzQ", - "MynPnB5r+rbMuc1M8ons7J7bx922DFre0HQ01RrHCE9zL8aqQ5XLG5oi+KIHX5htjGNDvFGme0ZjzlV/", - "yP4xIwzB3sEGk/ckBD6lzTR0fHEm0S2NY7B5gBEsC4Ahe11iBaa5VPq/ImNdNM4UEiThimiDM9F960Ey", - "mAs0HhOUMexObfpDVoaKXWAdryxYbohgJB7NCI6IkC0hYz5C9qNG4MBSJ1gqIgyHztIqvE7/fn6JOqcL", - "hhMaor+bXs95lMUEXWappuGtKvS6Q5YKMicMtF2tNVA7Lp8gnqken/SUIMRNMYHOcqvRHinMn11c2UMp", - "udUfsldEA5awSBudXCAnJSRSM6xQxNmfNcWSqNptefwa0P203A3mYZpVobxbh/ALOArS65lToTIca5ZV", - "Ubu8J0PmzNGjppojzbK6bFlRjnBYVV36bc0F0zMcQC4rz34LwSgczRbCmvNXn6M99zqEmVQ8KbnbUafm", - "UKBV10OVecx53IuwwqAatNRfzHSXj66ShenKbEoTlxxNxx4vlWaGlKEpneLxQlUV7p3B8tb7Ae3694G6", - "6VjXoAeJRoqvPtiiE+TatvFjwyHwSPHRfEI9PedCs/CgUInC2hmyRVrdRS8NqSXfLrqdUS1mJXJAAAq+", - "Pi8bgv0h6wHLOUKn+QB5t3mXmrOCtwy66HBRmgQFxycaL7YQRtfnffQ6n+2fJWJY0Tlx59wzLNGYEIYy", - "UM9IBOMDOy1PIJOah1FV/9zyKnMkvgX2Lrfv+kgbFAm2fF+jd4IVDcHZNqa19cAhh9koPZJmAKwsdVpJ", - "iVXHga/IlEolaoeBqPPq6cne3t6jur6w+6A32OntPHi9Mzga6P//s/254dc/9ff1dVzlF9Z9WeYoJ1dn", - "p7tWOamOoz7s40eH799j9eiA3spHH5KxmP62h+8kLsDPnk4LvyvqZJKInmN9Gqt83taSU7PBm/rZTtKN", - "QhLcscwq8WNW91q3/BZBDL6jNHuQs3mYQZ0Jrj2MKy1uaT36V60fFJhfchBYn3dIvd79UypvHguCb7RV", - "6ZGvWjzLkZE7fodXpu2o8QKR91o9IxESnKuJNP6Cqpqys/9w/3DvYP9wMPCc3S8jMQ/pKNRSpdUEXp6c", - "oRgviEDwDeqAoRehcczHVeR9sHdw+HDwaGe37TyMmdQODrkW5b5CHQuRv7g4MPemMqnd3YcHe3t7g4OD", - "3f1Ws7IKXqtJOWWwojo83Hu4v3O4u98KCj6z84mLpaifDUceJD1O05gaI7snUxLSCQ0RRGMg/QHqJCCW", - "SG7xVWlyjKORsGqgVx4oTGMPGEquPzOYbWlCb5IsVjSNiXkHG9JK04WVn0JPPjcxZYyIUR5qskFPNgJl", - "rWfMrSVvgiqRRBXQnVMJmkWhEFESR0eGQtfyOdjNYmJvmvDArqElNjznt0T0YjIncRkJjDjSk024ICjH", - "E7NplVVRNscxjUaUpZkXJRpB+TQToF+aThEe80wZUx02rDwInJuBjTDR7LrdsW3hqF4aWtuZGzr+UsEn", - "NPYsA4xW+9aKdOcSe74/uOzt/B/wg71k8cLwAcqMoZvwiPRrwYrQvvXyLprmlEeKovLsltaUuyY87tHc", - "2nUQsUZ3iBkaE2TFpHHqgtukGKRg8I98DHMicELG2WRCxCjxWFpP9XtkGhgfFGXo/HGVaWrm3Fbduqhs", - "DuhbExzaQL920PdYcrVldEvQfOPfrlfExDY0hRLorRK2jY0m6KMXeWwuenZxJVHhTvKYeC1P7S5mC6mN", - "E9OjiQyirGyZAXK2ZsMXxYfWhvUw48TLgBwhoM58mmZAhpevemcvr7eTiMy7lTmBC2jGY6LnvVXSreYu", - "oKA4YqwcucybVGSDGLItAZVglVNwayCV6NUDHcUVjkcy5sozm9f6JYKXqHP91Bwk6xl0UVrZSv17CQoV", - "/D7wUozmSE3DXsKAdVu7QuBr3R6JEVvl5VUG9ZHKLwTHJpK/is9FbJrbeH5T3Wh+s5Z6bSe+cc/cqVtN", - "ciYe2+Xk/NRYZiFnClNGBEqIwvbeQOl4G6Isgm7Q08pAhEkCPtHJf60+8G7w3eTossr6P1kKA/4mln9D", - "qJtmcvGcRCjBjE6IVDbUrTKynOHdBwdHJsg2IpP9Bwf9ft9/wqPEIuXUF+P4JH/Xbiu2zflor+izL2df", - "tg/f4CC/zVo+BhfHr38JjoLtTIrtmIc43pZjyo5Kf+d/Fi/gwfw5pswbANAqLptOluKxK9ubapllfj/S", - "K2EkzBGSg5a41jfpl+QvNGrG9AOJkDcsSuEp0vo3YNyXxT99QSRzcZ1GlSKYy8cELaKZ6YfV5rZTjKCN", - "HTNjisZFoPeyof1ZofpyZeTjUtRjSlge6xjH5inkbK6pwhf4WGHg7t3SZtxycUPZdBRRD3b+w7xEERUk", - "VBBXsp6Ggm2cputR0a/85TytbRC3DeHySJfvzsk/x+FaHf3l9G/v/q+8ePjbzrvn19f/PX/2t9MX9L+v", - "44uXXxRysjp677uG4K08UwMvYyX0ri16nGMVehSfGZeqAWr2DVIcJfrjPjoBA+1oyHroOVVE4PgIDQOc", - "0r4FZj/kyTBAHfIeh8p8hThDuit7dLylP74wYTf644/OBvxU7yOyZ8TCAjkP55DZOOIJpmxryIbM9oXc", - "QiQc2uinCIU4VZkgeke0rhkv0FjgsDgbLgbvoo84TT9tDRlYouS9EnoFKRYqD/V1I8BG21mZQyHbnERo", - "juOMSGvJDlkuP8A0150oLKZE9XMXIjhqagczDUDxmhlcVGMbDgddzz4i3U5vZEylIgzlXgkqAXlRxwWp", - "HA4q5H84OFx//pjj0Ar0A+xevlzrkLIFfRgEhqENMx7NlErXhy8AvzE0gn55/fpCg0H/e4lcRwUs8i02", - "xhhO05gSaU7VVAw6iY0L2gp8J2dmd1su6LVprD+LW4RhPIGB0evnl0gRkVBm+Hcn1OCc0FCvD853qJSZ", - "RkWK0fHJ+ZOtfovbwQDbfP4r9vF1vsLaMYJzbi1bmPBF4TTX8O2is9OuVqcshRaKFpybPuUCxYbBFHR9", - "hK4kqUYxwFaZIx6zk/Gi8JAZrj4MtlyPaZ1THKFXuX6H86nkVxAKZHBdFnQJ3drAFnOou9R7tzpXOK62", - "9otlbXCEixWyTm8Qxc2sYDX5eyAONM9Z3fe4GW2XnZZ6MD9qFHv/zTWQvU1tyU3DuatBaaUgxDyi+/uG", - "Yn9OYLXboWcXVxC+jOVIMpzKGVfNwRkYuTaIvKdSyeU4tlbhBMuB3FXxZEK0VwQGfs2QbJExBpER9WV8", - "m2Dr7xlw8OMFeq8Mzf7S+GqrpX2j8OpGruALTa4yCPPz1w2U/ibTqYQ8+zhCWZi5aLDPjnLuBtQTCXMs", - "JZ0yEqGzi+L+X+H1cN3X1vRot79zcNjfGQz6O4M2PqAEhyvGPj8+aT/4YNdYxUd4fBRGR2TyBT4oi9hG", - "68DxLV5INHR64TAwimhJAy2RrdUdW53vLQeTf17seF0SrosO3yQavF2Y94qL+ZfVK/mtlYsH//yi2/uk", - "rSy+hMbuq9Em3lGCQp7FEfuzQmNNecYeIJE1WyRRRbYDINYrdsP4Lasu3TjJNP2+y4hYoOvz84pLVZCJ", - "vfjdYuE8TRv3gacbbcPuGh1v7WxKEdd3EWVd54QlCfTVY6rL/h8X3GGwroUfqNABvWellBlw671fsaaa", - "BR+R+SjLfIqOfuXCNK+uzk4rG47xwc7h4PBR73C8c9DbjwY7Pbyzd9DbfYAHk73w4V5DipT2sRKfH/5Q", - "pdDmsGgAPHjDTCR7dKRpKI9fGGcK5TfVNHGeaI0RlZRREwQMBuoro5fqHkC6hvpNvMj11ZUfX2BNqO7b", - "FP5a/cXlLFNaDYJv5CxTSP8FU9ZLsPr+6i4MzR+hFxy+sTPtakFZMxxMc8yi8WK5ed3I6NgwEEGk4oJE", - "MJhlYEfoac60crZn2VxHEvtoeKkNl4JQsC1jVVsd3+5W0A0s1INuYEAYdAMHGf1oVghPMPmgG9iJeCMt", - "y3jj8xYTHAMPKyIxMkVj+sGQnJ46lYqGxs7CsJtNZGevs5FoZERo03mOOd63Yjb/yFH19TnqwOWDvyBr", - "hum/tvKznzIJ7e8+2n908HD30UGr0MViguu58QkEnyxPbi1rDtNs5FJFNSz95OIKhI8WbDJLTKykXXth", - "u2nGEWptjzJU5J4qBn/Uf1SO2Ix4No5L7gYbsg1hgW0ShTUcdryj8ZxOJuzdh/Bm9zdBk533B3J37DWO", - "8oH8muRZ2UW2ZHaRcc9cOvYH1QFCCdkYd/qKSFgBuiQKAf70NMPSEjWPGbEo56JTLcS9iLW/t7d3+PDB", - "biu8srMrEc4I7L/lWZ7bGZRIDFqizqvLS7RdQjjTpwukSwWRenHmKoWXztAwGwz2CBpUYuy07bHnw5IG", - "haXAGtv3PGkE+bXVWOyiLNAh9CXXZpao3Avtvb3Bw/0Hhw/akbG1eEbi/WoOY9vZI2NBQkLnlZ3vgFv1", - "9fEF0r2LCQ6rGv7O7t7+g4OHhxvNSm00KyUwkwlVaqOJHT48eLC/t7vTLoDa5zq1VwMqBFvlXR6i8yCF", - "Zzc8oFhmvd0maeHTEpfj7VaG+BUxg/UAsU0iQovrYFRCr7QUjIg6WokqK6SlK01bbfwMfhapx2lKQKnV", - "xbbBmqtjMy+wmp2xCV/2jW9i8NmIF3cSkWrFR0JqrogwSiLHu3LLz+pSEEMTS4KijFjIGd1IYAtwbM4H", - "UqxmoKzCh5RNq9HDSwO2McPMHFZf/oNxbcM2HiPpj9J4LTKAlfHqSoSLeI1WLmoqR36rYrljQaZZjAWq", - "BySvmLJcJDFlN216l4tkzGMaIv1B3Zyf8DjmtyP9Sv4V1rLVanX6g1FxNFkzz83k7MG02ZDauMUS/qpX", - "uVULdQHJv22+34YMw20ccN6Q3afaeDMxu1eMvi8hevUmzf7uoCmyqaHTSkzTcrz3przdoqyP4l0o9nGe", - "ccJzLmaObGoWbFUPrqzXt1o42loVx7WsCaCO8+m5m0pVuJZuDLUSxO1O1+reazebbUnC6uj7hw8eHrS8", - "svVFqvaKHKxfoFjPkxUKdcNOnbfR2g4fHD56tLf/4NHuRvqRO+ho2J+mw47y/tQSy9R0tgcD+N9GkzJH", - "Hf4pNRx3VCdUSRLz2RP6tIJ0i1sUDVb3qvznxU46M7+qgLdTcVdoS8cVlauUMKxDJhMCjqORgVuvmEwt", - "qqfVHEKc4pCqhccCxLcQ6IDyJrXbAC16r03WA1LbN8ITRQScRshsXFyH67jB0X8ay66GC4etb37KbNxk", - "Rb6sj2psSBMZFNU8FC0cBAYjfMfgtzkw0S2WFa++fg4VibqlhHD14x/Ton2+W4frecrb4mDbd6PFn962", - "vP217SxZHRUluQ7xVSK0mQS1RgBhR20c7B6J7LkmE64Po6jxBysAP++r0bh8J3vlpffKBe5C6m4+brsU", - "fMvfGQm2+XilE/xNPqxfTwV8tHOwIC/67lZQwodN5nylKfdJ4gqD1G6vUpNq3V5RQqXGqEOSVC1cGL6z", - "TLc2O+85zjv0IuNXDpwaPPoaodtXK2O1/4dk0ykfsblB1h6uLe1pY4CkX109rYevGJvQZhOohlvU7khL", - "taKiwKrqNaaMDBh8Njh5mtVvU21QsabJxC8ox5UKcCVr1lmuK/1ppZWVZtK8N+Z89QvL+1Dp6vp8Jsis", - "+bU+2tecUWkDuFdPN2EuqwoK9pwFkAGsBkFuoi/7AVaHfZzj9/kIYC1jiWoJ+sw6Shlvnz2GC+ivXNoB", - "OnFdwDTqqRYff1ndI4dVy5uxqhCSO8H3Ep7lPys4WhNt1ZCzGKO7utaSZl0kzARVi0stEGxwGsGCiOPM", - "oCFIClgE/FwMDhHvnz6BmTrxaKvPCCOChuj44gywJMEMT/WWXZ+jmE5IuAhjYgOWl8524b79y5Oznrlp", - "kWfEgzIGCgDiUlEdX5xBFhxbQCAY9Hf7kPWXp4ThlAZHwV5/B/L8aDDAErfhIhs8WkeUpkOQZGeRlbiP", - "TRMNWplyJg1wdgeDWkEKXGQa2f5NGg+LEa+tlUJT8Wc53mIpDtdpAnb6n7rB/mBno/msTQ7iG/aK4UzN", - "uKAfCEzzwYZA+KxBz5ixql1OYmIbFjgbHP1axdZf33x60w1kliRYq4gGXAWsUi6bVBgiEUaM3Nobjr/x", - "cR9dGpsEMoUUtdSMy4BEmiVhpLDoTz8gLMIZnZMhs5zYJHrBAq5zJEhzYBNMX0UzM7TZfUPCRKrHPFrU", - "oJt3t627A22kCuCNK3XkWQvThpIdPu5okiPJkHuzQhGGmSpy7ZisSDcEDjEn9L03IB7ie/3e7tP8navt", - "UuXtWt2lLIyzqBCA1Zoa3ovWpjaEzfN0Qzz6wjNoYedfDoV2kobxiJiw1nShZpyZ52ycMZWZ57Hgt5II", - "LY/svQwLFm025zW5TBI9msDdCHOTU4+5baa4/fGGLD71h+w4StzNW5vLFceS2wRYJkCBSpRnFB4yrwYt", - "R1j3Mxq74mI1RZVAV8NAi8phoJ+nAmuVLJMzhEMISNA/loHTMdjMBYi7rfpcQ8xQytMs1soDbI/JkFXp", - "A6644ThGCvDHfauFKMCkYT2ShIL4bKW/Xb58gYB/QpEVaFZEl8MaKNPSL88UqwfsD9kTHM6QEYyQQXEY", - "0GgYFMU0tkCIZZIY2dTrgWT9K1QZMsN0afTXfl93ZYT2Efr1o+nlSGNNmowUvyFsGHzqotKLKVWzbJy/", - "e9Ow4AZfzWUF5VHHMKQtdylYr7DEmw0zwyxC3DKAeIEwKmitbJKNKcNi0VSZhmeqOd7F3Jm2zYoLfQeD", - "wdb68wy7VI+6UmmoMfXTknTe/WqCyQrlZcFUqkKnxQCzF+IjI47vQDI+xpG7p/VTBVijAljbpSTc4Xur", - "AG5/pNEng74xMfGVNQkNxYqchE6xwAlRkKn6Vz/OQ2gp1X+700fwNRhLvoq83RJ46gr9myXE3m+sApXX", - "UwJc2L8D/INxizRlMO6juxoXxyZJbl6Z8l6hI2yWQ8Su3/p4RtSPgHGDu2KlLpvid8Tf+4I/z4hVkQqg", - "1bjZNqSnL5u29SsQguBE2l5MY23LXMKcepeEKQT1B2Xf/uvUbIgufxvz6dsjZEAY2+qL0ubHy33AWiha", - "WMJHJn1I/p3NqhPOMJsSiTpGfv7+r3+7CnK//+vftoLc7//6N5D7tq2HCt3ltQ/fHqG/E5L2cEznxC0G", - "IibJnIgF2hvYghzwypOjRw7ZkL0iKhNM5vFGel0AE9MhqOwM1kNZRiSSAELInj2xgTDGxeQx8RwtG1De", - "KUV3lyxdu4LSArRUdDgAJ5uUUUVxjHimTKJLmAdcyikmYtYclAeve8uW/Kfr+Ysi75XB3p6Z4IYMxtQO", - "9dCdKadp+kSdy8snW30E6r7BCgh2Aruh6MZaAv2fPGk9TzIcpcpQAMqGN5XSMzb62k5tm7twtjWlbmz2", - "tgnIM0+07eoW81PtbuF588PNeeF8rrBTl0682Rf2+ev1lRZtZVN+vX12uLcMc5srvwDZ97AmUcemOc6z", - "mVQS8n8vpL8TBlyq45BzYcRNDpU7s3BOOJvENFSo5+Ziy03mVk8VQe4LO3hlZ42wW1c9Qr8sKrYrAWeN", - "QiOPPbtL6VEbdBMxUtwiKHDtpyRZhzqnVIZcf1vCll6IUwCkBWJBp2UsWufbOYXfc5GzUjHPC8A6grw7", - "L48dOmN12XAHTPG0xhC/IyOspfko3bu5T9h8le+iK52ywgn0Y6Hm4O60oLt2CPnQ/D55hKIa2DQXnOXZ", - "xZvQy+Yf/4YbbUfwLPySCEfVZqImvUSxLPMpCmckvDELshV+VmkEZ64I0LfXA0wS9Q2kv53+T3HfwnAs", - "YLXKWDyzOUe+na0II2xkKn6940eLYB4gQ5TG2DlSTToPLBcs3PpDnUDeiWSoV+S5R5R0kcWxc8TPiVBF", - "KvkyP93+qPWDFnqyo7aVusjVq+c9wkIOMTkGdI0Kicsc/XW1ZbNhZik/0aSNfQWgcojRrIx+wf6b0CmU", - "Z3P80+5Tm8/xT7tPTUbHP+0dm5yOW98MWQZ3xZrvWnu9x8inlVdaBRqwJpPfeZ22l7e6E4XPJtLfROXL", - "J/hT62uj9ZXBtVLxy2safEPVz6aK/z7nBDmy+aANr1z82R9M5btb15PFyFL1v4ov3iY24aJIz25rh92/", - "ADmaY1yZ/7b0oRYEuVI7cKh7dtq1mfdNvvw8QPyOPKpuHneuJdpx796depyM6TTjmSwHtEOhBSKLqrQV", - "Bnzf9NdCPDdqsD8wlg7uUnTcuYL6E++/kepc31DDvG0h2zXKs2t1N8pzcVTTXnt2M/ypPbfSnkvgWq09", - "53lcv6X6bAb5bvqzwzcfwO0V5p8a9F1o0DKbTGhICVNFDqKlqBabwuwe3ith1glfOo2uMOHWGnSRXHm1", - "cmKR93tEIuSD373i7BKd3c/4WG4i4iOnqhbCsFlX/dHwYXC3zPnuddT7jGLPykXt/NqguRwS8+n6qyF5", - "T+4ehOduyJC5CnhvDVN/i3JERYojSWISKnQ7o+EM7ono36B/c40Ep+nb/GLo1hF6BvGn5auqMHhHEkFx", - "DBnTeWyS/b+dJ8nbo+WcEdfn5/CRuSJiskO8PUIuT0ROY1K3Kt/70KuIsVTohb3N0tEbLngcm+zMbzU8", - "S+vbsjdCiju0Q+a7HcLIre2QTtDb0kWRtw03RRwSPte79J0ov9ucHN+sRXEkAHDmzjphUcMtEQ01/x2R", - "nYE39VHL+ypmGt/4usrSZJ7zaZ5foILKOE3boq+dJmDxPElW4DDqlAoCSBXxTP1FqogIU7TWYncTcqMO", - "Ds0fCt+YEquVGnOmBIUPVPbutRdUgSkk7SpXmL/mSRKYgncJ9lWi+PJ7P/UOlw1GvTOlyz0/ZcYm13aq", - "zL50b6cmOWwJFMg24rUuX5kGf3jNxdWK+c5o+B0svWIWFErIsGi8gL0tivDcr0sLsJHFykDe2XV5acS9", - "a6QRW7vnD08jBX78wakk5AKqfktXgO/+RJeVLI4SuXeg4ldRSavrrN7r8/OtJqIxhaMbSUb8NIdtoOcf", - "XqZAEbT7Ry2m/ifOF7DKWagJQjXa6M5mrRRIHPNM976UPhUKg8iFVCQxBvski+HmHYTV2wQGuFz4pIuo", - "kpCGuwsuq1LRiyEbk4mWhykRemz9OaRnK2wPn1l7qXBOvheGBn8MuxYyqoIph1UT1GrVRdLUJVP12U55", - "/tfPntJTMFSrhVck6sT0xlQTRHOJYv2wtdLSNVVZvnZ6hs+nrLzukO/arcHZHJn/CBzurMbWXF3Ne8fW", - "npEysTj+AxvtZ2tyLV8TGxamdLArFajsD9k5UUK3wYKgkMcx1CMw+vt2Kni4DUXzwpRGpnoeTA4YXvPr", - "BEY8ubiCdiYFfHfI9B/LZdvqE3XV3862X67x/ZmCnf+D9RyzwFVk4d/wn26dzY8CGmlINpAoT1dp4jz9", - "qYjbOrw/zdZ7abbCWWy+ms5U4BCUYmkrLftNVFuebPujeThbd6KvcDi7dtUifgxt1yaXXzeMW+C9IEq7", - "poiYtAB3T5M8z/9/T69+acC5JYASU45N8EsBU1fkj4bdXz9OrgzHjaLk7pS2XMqNH4a27lry2Tm4QLUy", - "PO4LmRtMcyuBBOhl75MoFzhbaZu5+lNQbS9XLV3dtW65/J/J8Jn7kIq6MXmlsf6Q5aXVXIZRbV11nWmF", - "IipvTA/WeuojfwU8Y+fZMnhDpjgKcRyavPN5KThTvlE2WF+vSuURvxm9FYN4NjqvgSfzkmX3yeTw4wTs", - "XrkmGmCcVadWxqdf2zZ3EZ1uhdkGseluBT8j01tEppeA1aYCiyloZ7mVrUSWl8+AalD9hkIquVLy7eLa", - "P0Nefz30cHjaKK1/RrTfmUJQXAk9O73/Yexlmqvw6G1tFfRseaOya2gVBVsQpYL0XP2XyADMwsPYGvXq", - "Sf0hez0j7i9EXSgliWwF/XiBKIOCN64I3p8lEpyrosJ+c5UlQyJPBU+O7WrWGC+ty0H6DmI2zlfR9ZTA", - "o0mW5MXinz2G8tfCRPahCaYxxJU6kJL3ISGRBJzcqpeZ9Ib65fUk185yRYxmXkgqzKTiidv7s1PUwZni", - "vSlhei+Kmk2p4HMa1WsGV+p1+mYLFuJXMNKmH2haJb219W6WCa+KtygvUmUL7hT46XYn+Ckm6hmG9W5r", - "I88BUXGOYiymZOunKLnPoqTsTXJyoyJR2l2Iaudgaun3+RaXoXLn491ehbr+cXwipYys9zBhwDw3+pru", - "YP1YKDi4O/lw13evru+xD/0ZcQZu6d4VdKB79CHMcx7iGEVkTmKeQilq0zboBpmIbWHdo+3tWLebcamO", - "DgeHg+DTm0//PwAA//+YHpPWP+IAAA==", + "H4sIAAAAAAAC/+x97XLbOrLgq6C499bIdyRZ/ojj6NbUlmMnOZ6JE28ce/bOcVaBSEjCMQkwAChHSeXv", + "PMA84nmSLTQAfgmUqHw48ZxMTZ3QIgg0Go3+QqP7YxDyJOWMMCWD4cdAhjOSYHg8UgqHsyseZwl5Rd5l", + "RCr9cyp4SoSiBBolPGNqlGI1039FRIaCpopyFgyDc6xm6HZGBEFz6AXJGc/iCI0Jgu9IFHQD8h4naUyC", + "YbCdMLUdYYWDbqAWqf5JKkHZNPjUDQTBEWfxwgwzwVmsguEEx5J0a8Oe6a4Rlkh/0oNv8v7GnMcEs+AT", + "9Pguo4JEwfDX8jTe5I35+DcSKj340RzTGI9jckLmNCTLaAgzIQhTo0jQORHLqDg27+MFGvOMRci0Qx2W", + "xTGiE8Q4I1sVZLA5jajGhG6ihw6GSmTEg5kIYBrRyLMCx6fIvEanJ6gzI++rg+w+HB8GzV0ynJDlTn/J", + "Esx6GrkaLNc/tC33/Xzf1zPlSZKNpoJn6XLPpy/Pzi4RvEQsS8ZElHs83M37o0yRKRG6wzSkIxxFgkjp", + "n797WYZtMBgMhnh3OBj0Bz4o54RFXDSi1Lz2o3RnEJEVXbZCqe1/CaUvrk5PTo/QMRcpFxi+XRqpRthl", + "9JTnVSab6qr46P9xRuNomerH+mciRpRJhVkDDZ7alxpdfILUjCD7Hbo6Q50JFygi42w6pWy61YbeNcOK", + "iSLRCKvl4QBUZNtQzpCiCZEKJ2nQDSZcJPqjIMKK9PSbVgMKgtcMp1u0Gmx5q2VmJUeJbOrdNUGUoYTG", + "MZUk5CyS5TEoUwf7zZMpbRgiBPdwqCf6Z5QQKfGUoI5mm5p3MyQVVplEVKIJpjGJWq2RjxDMZH7jY0Qj", + "whSd0Or+NuTUw+NwZ3fPyzsSPCWjiE6tJKp2fwK/axLT/SgErf0T0Rtt0W4eMKQgk+XxngLrhkEEmRBB", + "NI1/4XCp4HPC9G7R4/0HjBv8r+1CRG9b+bwNyDwvmn/qBu8ykpFRyiU1EC5xLvtGkxGgGsEXfpjh1aq1", + "LlGUVFis3h/Q4ivsRANfK9xcmKZ1fgjsznZT2dmNbO/JnDCP4hNypuyL6oyf8ymKKSPItrD41XxOD/CX", + "mAOb+xpz6wYFSpc3tIb7MxiS+aGhN/2uGxCWJRqZMZ+WsTkjWKgxqSCzQSzZjgroGtF/XtkSNfmDJRmt", + "5grnlDESId3SblbTEmUStM+l6cPOuKFqNCdCevcRgPU3qpBt0dhVzMObCY3JaIblzECMowj2II7PKzPx", + "aGAVlRanmrG5DkEzkEhxdPHL0e6DA2QH8OBQ8kyEBoLlmZS+1t2btkhhMcZx7KWNZnLbXO4uU4ifAi7y", + "jdEkT3IKdIRpuFdgV1N33w3STM7ME/BjDRXIM80GNHnF+vmNZ9LHwCSM5t9oB/n1upepWWw0jbnG6QJl", + "jL7LKkpzH51q/V8hzfxpRKIuwvBCs2GcKd6bEkaE5lNoIngCGlRJsUUd0p/2u+ha63o9rdn28G5vMOgN", + "roOqahrv96ZpplGBlSJCA/j/fsW9D0e9fwx6j94Uj6N+782f/8NHAG21bafp2Xl23N7vIgdsWQWvA7pa", + "PV+h4fq4iFm+U733N12949NlAW/gj3h4Q0Sf8u2YjgUWi202pez9MMaKSFWdzeq2a+cHsK2YGJvqqW84", + "tZrBAeTWifktEaHmlDHRBCK7mllSJbsIa5sVmAzS0uy/UYiZplkj2LlAhEXolqoZwtCuioFk0cMp7VED", + "atANEvz+OWFTNQuGB3tL9KiJsWMfem/+y/209b+9JCmymHiI8RXPFGVTBK+N9J1RiQoYqCLJWnHrsJvF", + "oGIllJ2az3ZySLAQeOFfNQfcqtUzxlHj8oWJR5N+OSdC0MhJtOOzE9SJ6Q2xZIlExtB1NhjshdAAHon9", + "JeRJgllkftvqo5cJVVqSZIWANN6VfnkJfw1IOOMg4+OY6wnl6GtQIBxenKHpWaIT55mQyFq7INMw+J1g", + "yZ6dX25rrpJiKdVM8Gw6q0JlWdpm8FB5M6J8NE59MFF5g063XyLNcFFMNXZyBrszGJw93pbXgf7jgftj", + "q49ODMoAfL1+XFi+L2dYENA+IsQZOj6/RDiOeWjtuYlWEid0mgkS9WtuBOjdR/CEKbFIOfUpnzXKKJou", + "E0ivV7zdgA62x5RtS70MvXAzvBM2/wIV6AmbU8FZotXQORZU862KU+dj8OLlyZPRkxdXwVBvoigLrYfk", + "/OWr18Ew2BsMBoFPy9AUtIYPPDu/PIaV0u1nXKVxNh1J+oFU3JHB3rPHQR3wo3y+KCEJF8YUsH2gzqzK", + "iY2mhGCxrnV/hth2ntVl5C4MtYS02SIlYk6lz8T/JX/nFrrEFg0fqpKyJGJORE6jQLT9kpoVxjyLeqUh", + "u8E7kmgNY/5B00YBrael39ZuJYXXiFccp5SRRvnaDRKiMPiZP58cLyURvYhMqLYubsiiN8dxRpDr2WKW", + "5IitUmqaiZRL0z+eGq1UEZwEw2CMwxvCIi+h/iCy/JaLm5jjqLfzlUU5I0r3vTzFF+ZFlRJ9OK7bhyy6", + "pZGajSJ+yzTIHoZv36C8cc713+uZ4Pj3f/7r6qxQXHeejVMrAnZ2H3yhCKgxfd211yjNJ5Kl/mlcpv5J", + "XJ39/s9/uZl830kQpukzqnBM4+epTuXvM6JmRJRUAbfA+idjVcDnyNFLafiK46h82rO0mficiBgvPFx8", + "Z+Bh438XVMH+st8hrUYg/fEaHq57cxrDMhcf+Nm4BygPTI/1/rZCpQ0kOSA7u2f2cbetYJE3NB1NtZI6", + "wtPc8bXqHO7ihqYIvujBF2YZ49hs3ijTPaMx56p/zf4+IwzB2sECk/ckBD6lLXt0dH4q0S2NYzCTgREs", + "C65r9rrECkxzqfR/Rca6aJwpJEjCFUFWA4ZBMoAFGo8Jyhh2B339a1bGip1gna4sWm6IYCQezQiOiJAt", + "MWM+QvajRuTAVCdYKiIMh87SKr5O/nZ2gTonC4YTGqK/mV7PeJTFBF1kqd7DW1Xsda9ZKsicMDCQtLZD", + "7bh8gnimenzSU4IQB2ICneWOBnsKNX92fmnPMeVW/5q9IhqxhEUkApidlJBIzbBCEWd/0jsWxGWp2/L4", + "NaT793I3mIdpVsXybh3DL+D0UM9nToXKcKxZVkVd9B4mmmNqj1lgTsHL5ollRTnBYVU9BWprYZqe4cx6", + "WWn2G5VGUWo2Ktcc2fvOZnJHVZhJxZPSCQ3q1HxQtOqtqjKPOY97Wv8B1WBZvnv1FwPu8mlnsjBdmUVp", + "4pKj6djj2NTMkDI0pVM8XqiqobAzWF56P6Jd/z5UN0UCGPIg0Ujx1WehdIJc2zZHHxA3MFJ8NJ9QT8+5", + "0CycblSisBZ2YIlWd9FLQ2q3bxfdzqgWsxI5JMAOvjorG979a9YDljNEJ/kAebd5l5qzgoMVuuhwUQKC", + "gq8cjRdbCKOrsz56nUP7J4kYVnROXGjEDEs0JoShDNQzEsH4wE7LAGRS8zCq6p9bXmWiKLbAv8Dtuz7S", + "hlCCLd/X5J1gRUPwz45pbT5wLmYWSo+kGQArS51WUmLVCfIrMqVSidr5Meq8enq8t7f3qK4v7D7oDXZ6", + "Ow9e7wyGA/3/f7Q/av76gSK+vo6q/MJ6vMsc5fjy9GTXKifVcdSHffzo8P17rB4d0Fv56EMyFtPf9vCd", + "hJL42dNJ4apHnUybfY71aaryOehLfvAGB/xn+9U3imJxJ3mrxI+Z3Wvd8lvEvfhOX+3Z3+aRKXUmuPb8", + "tjS5pfnoX7V+UFB+ybFhj0lC6j0QOqHy5rEg+EZblR75qsWzHBm543cwZtqOGi8Qea/VMxIhwbmaSOPn", + "qKopO/sP9w/3DvYPBwNPuMcyEfOQjkItVVoB8PL4FMV4QQSCb1AHDL0IjWM+rhLvg72Dw4eDRzu7beEw", + "ZlI7PORalPsKdSxG/uxCB92bClC7uw8P9vb2BgcHu/utoLIKXiugnDJYUR0e7j3c3znc3W+FBZ/Z+cSF", + "39TDCSIPkR6laUyNkd2TKQnphIYIAniQ/gB1EhBLJLf4qntyjKORsGqgVx4oTGMPGkouSzOYbWmitZIs", + "VjSNiXkHC9JK04WZn0BPPvcwZYyIUR6dtEFPNmhprWfMzSVvgirBZxXUnVEJmkWhEFESR0OzQ9fyOVjN", + "ArA3TXRg59CSGp7zWyJ6MZmTuEwERhxpYBMuCMrpxCxaZVaUzXFMoxFladbgGW1A5dNMgH5pOkV4zDNl", + "THVYsPIgcNQKNsJEs+t2J/2Fg31paG1nbuj4SwWf0NgzDTBa7Vsr0p1L7Pn+4KK383/AD/aSxQvDBygz", + "hm7CI9KvxbdC+9bTO2+CKQ8uRmXoluaUuyY87tHc2nUYsUZ3iBkaE2TFpHHqgtukGKRg8I98DHMicELG", + "2WRCxCjxWFpP9XtkGhgfFGXo7HGVaWrm3FbdOq8sDuhbExza2NB22PdYcrVpdEvYfONfrlfEhMM0RZ/o", + "pRK2jQ1A6aMXeTg3enZ+KVHhTvKYeNXlbTwlPZ8tpDZOTI8mmIyysmUGxNmaDZ8XH1ob1sOMEy8DchsB", + "debTNINtePGqd/ryajuJyLxbgQlcQDMeEw33Vkm3mrsYlOJIt3JUNG9SkQ1hyLYbqISrfAe3RlJpv3qw", + "o7jC8UjGXHmgea1fIniJOldPTeyBhqCL0spS6t9LWKjQ94F3x2iO1DTsBQxYt7UrG3yt2yMxYqs8vcqg", + "vq3yC8GxufxRpecinNEtPL+pLjS/Wbt7bSe+cU/daWGLeInjsxNjmYWcKUwZEflBXfVwGwJzgm7Q08pA", + "hEkCPtHJf68+6G7w3eTkssr6P16KHP8mln9DdKRmcvGcRCjBjE6IVDY6sjKynOHdBwdDE5cdkcn+g4N+", + "v79pZMKTIhSh1VJsm3PdUpBCX86+bB2+QQBCm7l8DM6PXv8SDIPtTIrtmIc43pZjyoalv/M/ixfwYP4c", + "U+Y9D24Vyk8nSyH8leVNtcwyvw/1TBgJc4LkoCWu9U36JfkLTZox/UAi5I2kU3iKtP4NFPdlIXNfEPxe", + "3MBSpaD38jFBiwB4+mG1ue0UI2hjx8yYonFxN2DZ0P6s2x1yZbDsUqBsSlgeHhvH5inkbK53hS9WtsLA", + "3bulxbjl4oay6SiiHur8u3mJIipIqCAeZv0eCrZxmq4nRb/yl/O0tnH/NurPI12+Oyf/HIdrdfSX07++", + "+7/y/OFvO++eX139z/zZX09e0P+5is9ftj+y8YScrA74/K5RmyvP1MDLWInWbEseZ1iFHsVnxqVqwJp9", + "gxRHif64j47BQBtesx56ThUROB6i6wCntG+R2Q95ch2gDnmPQ2W+Qpwh3ZU9Ot7SH5+bsBv98UdnA36q", + "9xHZM2JhkZyHc8hsHPEEU7Z1za6Z7Qu5iUg4tNFPEQpxqjJB9IpoXTNeoLHAYXE2XAzeRR9xmn7aumZg", + "iZL3SugZpFioPDrcjQALbaEyh0K2OYkQxFVJa8les1x+gGmuO1FYTInq5y5EcNTUDmYakOI1M7ioxjYc", + "DrqedUS6nV7ImEpFGMq9ElQC8aKOC1I5HFS2/+HgcP35Y05DK8gPqHv5PrYjyhb7wxAwDG2Y8WimVLo+", + "fAH4jdkj6JfXr881GvS/F8h1VOAiX2JjjOE0jSmR5lRNxaCT2LigrcB3cmZWt+WEXpvG+rO4RRjGExgY", + "vX5+gRQRCWWGf3dCjc4JDfX84HyHSplpUqQYHR2fPdnqt7hQDrjN4V+xjq/zGdaOEZxza9nChC8Kp7nG", + "bxednnS1OmV3aKFowbnpUy5QbBhMsa+H6FKSahQDLJU54jErGS8KD5nh6tfBlusxrXOKIXqV63c4ByW/", + "tVIQg+uy2JfQrQ1sMYe6S713q7DCcbW1XyxrgyNcrJB1eoMobmYFq7e/B+Ow5zlrDOxstbfLTks9mJ80", + "irX/5hrI3qa25Kbh89WgtFIQYh5B3z70/VuEkC/bVe+pGjWeyCD92p6/OOvh6gzNsGR/UvCyZkPs7D1s", + "dTFbj9r2LKN8isEnBqR8V7kIt9wHb2L9bmgcm6MtSacMx+gR6lycPvvb6fPnW6iHXr48qy/Fqi9869Mi", + "kt6R9rPzS4hXx3IkGU7ljKvmqBaMXBs9U6nkcgBgqziM5cj9qlw3MfkrIiq/Zgy+yBiDkJL6NL5NdP33", + "jNT4N4rsD74kLn9lJP2XhsNbpfobRcM3MnFfJHmVn5ufv25c+zcBpxKh7uNDZd3DBe99dlB6N6CewKUj", + "qVktidDpeXHDt3BSue5rc3q02985OOzvDAb9nUEbl12CwxVjnx0dtx98sGucGEM8HobRkEy+wGVoCdso", + "iTi+xQuJrp0afx0Yu6FkMJSYhVX1Wx3HLsf+f16of11xWRfMv0nwfruo/BWpNy6qSTda64IP/vFF+TlI", + "Ww3gAhq7r0abOLMJCnkWR1rfGuudZ8w3ElkrUxJV5DOBzXrJbhi/ZdWpG5+m3r/vMiIW6OrsrOIBF2Ri", + "Uzu0mDhP08Z14OlGy7C7RiVfC00pQP4uguLrnLAkgb56CHzZXedicQzVtXDbFZqn92ibMoNuvfYr5lRz", + "uERkPsoyn3qlX7mo2svL05PKgmN8sHM4OHzUOxzvHPT2o8FOD+/sHfR2H+DBZC98uNeQBKl9aMvnR6tU", + "d2hzFDsgHpyX5uJBNNR7KA83GWcK5RcL9eY81noqKqnAJmYb/AmvjDasewDpGuo38SLXkld+fI71RnXf", + "pvDX6i8uZpnSahB8I2eZQvovAFlPwVoZq7swe36IXnD4xkLa1YKyZq6Y5phF48Vy87pp07FRO4JIxQWJ", + "YDDLwIboac60crZn2VxHEvtoeKmNboPIvS3jBLGWhV2toBtYrAfdwKAw6AYOM/rRzBCeAPigG1hAvIGx", + "ZbrxOfcJjoGHFYEzmaIx/WC2nAadSkVDY91hWM2mbWdvH5JoZERo0/GbicawYjb/yO3qqzPUgbsif0bW", + "+NN/beVHdeUttL/7aP/RwcPdRwetIk0LANdz42OIFVoGbi1rDtNs5JLBNUz9+PwShI8WbDJLjDVv515Y", + "jJpxhFrbowwV2eWKwR/1H5UDbCOejeOSd8hG2EMUZ5tUgA1nU+9oPKeTCXv3IbzZ/U3QZOf9gdwde42j", + "fCC/Jnla9mgumV1k3DN32/02JBCUkI1hwq+IhBmgC6IQ0E9PMywtUfMQH0tyLpjYYtxLWPt7e3uHDx/s", + "tqIrC11p44zA/luG8sxCUNpi0BJ1Xl1coO0SwZk+XdxjKojUkzM3X7z7DNmUIoNKSKS2PfZ8VNKgsBRU", + "Y/ueJ40ov7Iai52URTpEKuXazNIu92J7b2/wcP/B4YN229haPCPxfjWHse3sCb8gIaHzysp3wAv++ugc", + "6d7FBIdVDX9nd2//wcHDw42gUhtBpQRmMqFKbQTY4cODB/t7uzvt4t19nm57k6OyYau8y7PpPEThWQ0P", + "KpZZb7dJWvi0xOXwyJURmUWIZz2eb5MA3uL2HpXQKy3FjqKOVqLKCmnpBtpWGz+Dn0XqcZpSzGp1sW1s", + "7epQ2nOsZqdswpePMjYx+GyAknNxp1rxkZB8LyKMksjxrtzys7oUhDzFkqAoIxZzRjcS2CIcm+OcFKsZ", + "KKvwIWXTarD30oBtzDADw+q7mjCubdjGYyT9QTWvRQa4Mr5kiXARXtPKMU7lyG9VLHcsyDSLsUD1+PEV", + "IMtFElN206Z3uUjGPKYh0h/UzfkJj2N+O9Kv5F9gLlutZqc/GBUnyTXz3ABn4wjMgtTGLabwFz3LrVpk", + "Ekj+bfP9NuQQb+OA8x4vPdXGmwmxvmT0fYnQqxef9ncHTYFoDZ1WQtCWw/M35e2WZH073kXOH+UJQjzH", + "mOagqGbBVvXgynx9s4WTyFVhd8uaAOo4n567WFbFa+mCVytB3O4wtO69dtBsSxJWR98/fPDwoOUNuy9S", + "tVdkWf4CxXqerFCoG1bqrI3Wdvjg8NGjvf0Hj3Y30o/cQUfD+jQddpTXp5YHqKazPRjA/zYCyhx1+EFq", + "OO6oAlTJ6fPZAH1asXWLSy8NVveqCgfFSjozv6qAt1NxV2hLRxWVq5RPr0MmEwKOo5HBW68AphaE1QqG", + "EKc4pGrhsQDxLcSloLxJ7fJGi95rwHpQavtGeKKIgNMImY2Lc/+OGxz9l7HsarRw2PqirszGTVbky/qo", + "xoY0gVxRzUPRwkFgKMJ3+H6bIxPdYlnx6uvnUJGoW8qXWD/+MS3aZ7R2tJ4ntS6O030XkPwJrMvLX1vO", + "ktVRUZLrGF8lQpu3oNYIIEqsjYPdI5E9t5rC9cEbNf5gBeDnfTUal6/Qr8xRULlvX0jdzcdtl+lx+Tsj", + "wTYfr3SCv8mH9dvEQI8WBovyou9uhSR81GTOV5pS1SSu9E/tsjE1xRTsjTJUaow6JEnVwt2acJbp1mbn", + "PUd5h15i/MpxboNHXyPS/nJlaP2/SfKj8hGbG2Tt4drSmjbGs/rV1ZN6+IqxCW3yh2q4Re1Ku1Qraoas", + "qk9lCkWBwWdjyadZ/fLbBjWpmkz8Yue4YiCuKNU6y3WlP600sxIkzWtjzle/sIAXla5y12eizJpf64Oz", + "zRmVNoB79ewg5m6xoGDPWQQZxGoU5Cb6sh9gddjHGX6fjwDWMpaolk/RzKOUEPrZY8gX8MpliaAT1wWA", + "Uc+M+fjLKps5qlpejFWlztwJvnfjWf6zgqM17a0acRZjdFdXU9Osi4SZoGpxoQWCDU4jWBBxlBkyBEkB", + "k4Cfi8HhgsKnT2CmTjza6jPCiKAhOjo/BSpJMIP8uujqDMV0QsJFGBMbX750tgvpEV4en/bMxZg8gSEU", + "KlGAEJc57Oj8FJIW2RIhwaC/24fk0jwlDKc0GAZ7/R1Iy6TRAFPchnuH8GgdUXofgiQ7jazEfWyaaNTK", + "lDNpkLM7GNRKzuAiMcz2b9J4WIx4ba0Umppey/EWSwGRThOw4H/qBvuDnY3gWZvLxTfsJcOZmnFBPxAA", + "88GGSPisQU+Zsapd6mtiGxY0Gwx/rVLrr28+vekGMksSrFVEg64CVymXTSoMkQgjRm7thdTf+LiPLoxN", + "AtHlRbVE4zIgkWZJGCks+tMPCItwRufkmllObPLyYAG3bxKkObC5+1AlMzO0WX2zhYlUj3m0qGE3725b", + "d9dzUbUFgjeuxZMnmUwbivL4uKPJZSVD7k3iRRhmqkiNZJJY3RA4xJzQ9977CxBV7Pd2n+TvXPWmKm/X", + "6i5lYZxFhQCsVs3x3os31V9sWq4b4tEXnkELC385ANtJGsYjYsJa04WacWaes3HGVGaex4LfSiK0PLLX", + "aCxatNmcV90zOQ9pAldZzMVbPea2AXH74w1ZfOpfs6MocRelbepdHEtu85WZAAUqUZ4A+pp5NWg5wrqf", + "0diVD6wpqgS6ug60qLwO9PNUYK2SZXKGcAgBCfrHMnI6hpq5AHG3VYc1xAylPM1irTzA8piEZpU+4EYi", + "jmOkgH7ct1qIAk4a5iNJKIjPVvrrxcsXCPgnlFGCZkVMO8yBMi398sS+esD+NXuCwxkyghESXl4HNLoO", + "inI5WyDEMkmMbOr1QLL+BeqImWG6NPpLv6+7MkJ7iH79aHoZaqpJk5HiN4RdB5+6qPRiStUsG+fv3jRM", + "uMFXc1EhedQxDGnL3eHWMyzxZsPMMIsQtwwgXiCMir1WNsnGlGGxaKo9xTPVHO9irrjbZsX9y4PBYGv9", + "eYadqkddqTTUlPppSTrvfjXBZIXysmAq1ZnUYoDZ/AWREcd3IBkf48hdq/upAqxRAaztUhLu8L1VALc/", + "0uiTId+YmPjKmoSGcmROQqdY4IQoSCz+q5/mIbSU6r/d6SP4GowlXyXebgk9dYX+zRJh7zfWecsrpgEt", + "7N8B/cG4RVY5GPfRXY2LY5PTOK89e6/IERbLEWLXb308I+pHoLjBXbFSl/zyO9LvfaGfZ8SqSAXSatxs", + "G6oJlE3b+hUIQXAibS+msbZlLgCm3gVhCkGFUdm3/zo1G6LL38Z8+naIDApjW19V2nSGuQ9YC0WLS/jI", + "ZHvJv7NJkMIZZlMiUcfIz9//+S9XI/L3f/7L1oj8/Z//gu2+bSseQ3d5ddO3Q/Q3QtIejumcuMlAxCSZ", + "E7FAewNbPwVeeVIqyWt2zV4RlQkm83gjPS/AiekQVHYG86EsIxJJQCEkO5/YQBjjYvKYeG4vG1Te6Y7u", + "Llm6dgalCWip6GgATjYpo4riGPFMmbykAAdcyikAMXMOyoPXvWVL/tP1/EWR98pQb88AuCGDMdWBPfvO", + "FMw1faLOxcWTrT4Cdd9QBQQ7gd1QdGMtgf5PnrSeJxmOUmUogGXDm0rZNBt9bSe2zV0425oybTZ72wSU", + "BSDadnWT+al2t/C8+fHmvHA+V9iJy/7e7Av7/Pn6ige3sim/3jo72lvGuS1tUKDse1iTqGOzUufJZyr1", + "E74X0d8JAy6V3ci5MOIm5c2dWTjHnE1iGirUc7DYaqy51VMlkPvCDl5ZqBF286pH6JdFxXYl4KxRaOSx", + "Z3cpPWqDbiJGilsEBa39lCTrSOeEypDrb0vU0gtxalPvABKLfVqmonW+nRP4PRc5KxXzvD6y25B35+Wx", + "Q2esLhvugCme1Bjid2SEtTQfpXs394maL/NVdJVuVjiBfizSHNydFnTXDiEfmd8nj1BUQ5vmgrM8GXwT", + "edl08d9woe0InolfEOF2tQHUpJcopmU+ReGMhDdmQrYg0yqN4NTVbPr2eoDJeb+B9Lfg/xT3LQzHAler", + "jMVTm3Pk29mKMMJGpuLXO360BOZBMkRpjJ0j1aTzwHLBwq0/1AnknUiGegGle7STzrM4do74ORGqyPxf", + "5qfbH7V+0EJPdrttpS5y+ep5j7CQQ0yOQV2jQuISfX9dbdksmJnKTzJpY18BqhxhNCujX7D+JnQK5Tkk", + "/3P3qc0i+Z+7T00eyf/cOzKZJLe+GbEM7oo137X2eo+JTyuvtIo0YE0mHfc6bS9vdScKn617sInKlwP4", + "U+tro/WV0bVS8ctLUHxD1c9m9v8+5wQ5sfmwDa9c/NkfTOW7W9eTpchSscaKL94mNuGiyKZvS73dvwA5", + "mlNcmf+29KEWG3KlduBI9/SkawslmPIGeYD4HXlUHRx3riXace/enXqUjOk045ksB7RDXQwiiyLCFQZ8", + "3/TXQjw3arA/MJUO7lJ03LmC+pPuv5HqXF9Qw7xt3eE1yrNrdTfKc3FU0157dhD+1J5bac8ldK3WnvM8", + "rt9SfTaDfDf92dGbD+H2CvNPDfouNGiZTSY0pISpIgfRUlSLTWF2D++VMOuEL51GV5hwaw26SK68Wjmx", + "xPs9IhHywe9ecXaJzu5nfCw3EfGRU1ULYdisq/5o9DC4W+Z89zrqfSaxZ+UahH5t0FwOifl0/dWQvCd3", + "D8JzN+SauYKFbw1Tf4tyQkWKI0liEip0O6PhDO6J6N+gf3ONBKfp2/xi6NYQPYP40/JVVRi8I4mgOIaM", + "6Tw2yf7fzpPk7XA5Z8TV2Rl8ZK6ImOwQb4fI5YnI95jUrcr3PvQsYiwVemFvs3T0ggvuCq691fgszW/L", + "3ggp7tBeM9/tEEZubYd0gt6WLoq8bbgp4ojwuV6l77Tzu83J8c1cFEcCEGfurBMWNdwS0Vjz3xHZGXhT", + "H7W8r2LA+MbXVZaAec6neX6BCinjNG1LvhZMoOJ5kqygYdQpFQSQKuKZ+rNUERGmxrCl7ibiRh0cmj8U", + "vjEVcSuV7UwJCh+q7N1rL6oCU/fbVa4wf82TJDBl9hLsq0Tx5fd+6h0uG4x6ZUqXe37KjE2u7VSZfene", + "Tk1y2BIokG3Ea12+Mg3+8JqLqxXzncnwO1h6BRQUSsiwaLyAtS2K8NyvSwuwkMXMQN7ZeXn3iHvXuEds", + "7Z4//B4p6OMPvktCLqBIu3QF+O5PdFnJ4iht9w5U/CoqaXWd1Xt1drbVtGlMne/GLSN+DHP483yltWT7", + "SeTP6yxo5BJwHZ+dFDWVRcb66GVCIT3TDSEpXGanPJMIiuH2y5leG9ItF6lcCVNikXLK1FooiqbfBphP", + "n5Xe5475lI2v/cOLcqg9d/+YlCm7ivMJrPLRaj6kGl0jzlVQqUs55pnufSlrLdRjkQupSGL8JJMshk0E", + "txls3ghcrjfTRVRJyH7eBU9hqdbINRuTiVZDUiL02PpzyIpXmHw+b8KFwjnXPDes78dwJ0AiW7CgsWrC", + "Wq2oS5q6HLY+kzVPu/vZID0F/0C13o1EnZjemCKOaC5RrB+2VjoYTDGcr50V4/N3Vl7uyXfb2dBsTsx/", + "BA53WmNrrpzpvWNrz0h5szj+AwvtZ2tyLV8TG9YDdbgr1QXtX7MzooRugwVBIY9jKANhzKbtVPBwG2oV", + "himNTNFCAA4YXvPrBEY8Pr+Edibzfvea6T+Wq+XVAXVF9063X65xuZo6qf/G9piZ4Kpt4V/wn960zU9g", + "GveQbNiiPF1lAPH0D+8wsBrcT2/B/fQWwBF4PpvOVOAQlGJpC1z7PQO2Ktz2R/Nwui6QQuFwduWKdPwY", + "2q7N6b9uGDfBe7Ep7ZwiYrIx3P2e5HnZhXt6404jzk0BlJhySIhfCphyLn806v764YllPG4UnHine8tl", + "Ovlh9tZdSz4Lg4sPLOPjvmxzQ2luJpB3vux9EuW6cittM1f2C4oc5qqlK3fXLVddNIlVcx9SUa4nL/DW", + "v2Z5RTuX2FVbV11nWqGIyhvTg7We+shfeNDYebb64DVTHIU4Dk26/7wCn6maKRusr1elqpTfbL8Vg3gW", + "Oi89KPNKcffJ5PDTBKxeuRQdUJxVp1ZeC7iybe7iUoAVZhtcCXAz+HkhoMWFgBKy2hS+MXUELbeyBeDy", + "qiVQhKvfUL8mV0q+3XWCz5DXX488HJ02SuufFwnuTCEobuKentz/2wPlPVfh0dvaKujZqlJl19CqHWxR", + "lArSc2V3IoMwiw9ja9SLVvWv2esZcX8h6iJYSYQiKkio4gWiDOoMudqDf5JIcK7sey4WzcWtzBZ5Knhy", + "ZGezxnhpXYXTdxCzcZqQrqfyIE2yJK/R/+wxVB0XJqASTTCNIZzXoZS8DwmJJNDkVr26pzfCMi/juRbK", + "FaGxef2uMJOKJ27tT09QB2eK96aE6bUoSmWlgs9pVC/VXCmT6oMWLMSvYKRNP9C0uvXWlhla3nhVukV5", + "bTBb56igT7c6wU8xUU/srFdbG3kOiYpzFGMxJVs/Rcl9FiVlb5KTGxWJ0u4eWjsHU0u/z7e4g5Y7H+/2", + "BtrVj+MTKSXCvYd5Gua50dd09e3HIsHB3cmHu77ydnWPfejPiDNwS9fdoAPdo49gnvMQxygicxLzFCqA", + "m7ZBN8hEbOsZD7e3Y91uxqUaHg4OB8GnN5/+fwAAAP//elCOqJjnAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/system/README.md b/lib/system/README.md index 30afa646..25a5e19f 100644 --- a/lib/system/README.md +++ b/lib/system/README.md @@ -70,9 +70,13 @@ It replaces the previous shell-based init script with cleaner logic and structur - ✅ Hand off to systemd via chroot + exec (systemd mode) **Two boot modes:** -- **Exec mode** (default): Init chroots to container rootfs, runs entrypoint as child process, then waits on guest-agent to keep VM alive +- **Exec mode** (default): Init chroots to container rootfs, runs entrypoint as child process. When the app exits, init logs exit info and cleanly shuts down the VM via `reboot(POWER_OFF)`. - **Systemd mode** (auto-detected on host): Init chroots to container rootfs, then execs /sbin/init so systemd becomes PID 1 +**Graceful shutdown:** The host sends a `Shutdown` gRPC RPC to the guest-agent, which signals PID 1 (init). Init forwards the signal to the entrypoint child process. If the app doesn't exit within the stop timeout, the host falls back to a hard hypervisor kill. + +**Exit info propagation:** When the entrypoint exits, init writes a machine-parseable `HYPEMAN-EXIT` sentinel to the serial console with the exit code and a human-readable description (signal names, OOM detection via `/dev/kmsg`, shell conventions for 126/127). The host lazily parses this from the serial log when it discovers the VM has stopped, and persists `exit_code`/`exit_message` to instance metadata and the API. + **Environment variables:** In exec mode, env vars are passed directly to the entrypoint and guest-agent processes. In systemd mode, env vars are written to `/etc/hypeman/env` and loaded via `EnvironmentFile` in the `hypeman-agent.service` unit. **Systemd detection:** Host-side detection in `lib/images/systemd.go` checks if image CMD is diff --git a/lib/system/guest_agent/shutdown.go b/lib/system/guest_agent/shutdown.go new file mode 100644 index 00000000..e29690ae --- /dev/null +++ b/lib/system/guest_agent/shutdown.go @@ -0,0 +1,28 @@ +package main + +import ( + "context" + "log" + "syscall" + + pb "github.com/kernel/hypeman/lib/guest" +) + +// Shutdown sends a signal to PID 1 (init) to trigger graceful shutdown. +// The guest-agent is the messenger -- init owns the process lifecycle and +// will forward the signal to the entrypoint child process. +func (s *guestServer) Shutdown(ctx context.Context, req *pb.ShutdownRequest) (*pb.ShutdownResponse, error) { + sig := syscall.SIGTERM + if req.Signal != 0 { + sig = syscall.Signal(req.Signal) + } + + log.Printf("[guest-agent] shutdown requested with signal %d (%s)", sig, sig.String()) + + if err := syscall.Kill(1, sig); err != nil { + log.Printf("[guest-agent] failed to signal PID 1: %v", err) + return nil, err + } + + return &pb.ShutdownResponse{}, nil +} diff --git a/lib/system/init/config.go b/lib/system/init/config.go index 35c9786b..7ab780fa 100644 --- a/lib/system/init/config.go +++ b/lib/system/init/config.go @@ -30,7 +30,7 @@ func readConfig(log *Logger) (*vmconfig.Config, error) { if output, err := cmd.CombinedOutput(); err != nil { return nil, fmt.Errorf("mount config disk: %s: %s", err, output) } - log.Info("config", "mounted config disk") + log.Info("hypeman-init:config", "mounted config disk") // Read and parse config.json data, err := os.ReadFile(configFile) @@ -51,6 +51,6 @@ func readConfig(log *Logger) (*vmconfig.Config, error) { cfg.Env = make(map[string]string) } - log.Info("config", "parsed configuration") + log.Info("hypeman-init:config", "parsed configuration") return &cfg, nil } diff --git a/lib/system/init/headers.go b/lib/system/init/headers.go index eacba01a..1cf330f1 100644 --- a/lib/system/init/headers.go +++ b/lib/system/init/headers.go @@ -25,23 +25,23 @@ func setupKernelHeaders(log *Logger) error { return fmt.Errorf("uname: %w", err) } runningKernel := int8ArrayToString(uname.Release[:]) - log.Info("headers", "running kernel: "+runningKernel) + log.Info("hypeman-init:headers", "running kernel: "+runningKernel) // Check if headers tarball exists in initrd if _, err := os.Stat(headersTarball); os.IsNotExist(err) { - log.Info("headers", "no kernel headers tarball found, skipping") + log.Info("hypeman-init:headers", "no kernel headers tarball found, skipping") return nil } // Clean up mismatched kernel modules directories if err := cleanupMismatchedModules(log, runningKernel); err != nil { - log.Info("headers", "warning: failed to cleanup mismatched modules: "+err.Error()) + log.Info("hypeman-init:headers", "warning: failed to cleanup mismatched modules: "+err.Error()) // Non-fatal, continue } // Clean up mismatched kernel headers directories if err := cleanupMismatchedHeaders(log, runningKernel); err != nil { - log.Info("headers", "warning: failed to cleanup mismatched headers: "+err.Error()) + log.Info("hypeman-init:headers", "warning: failed to cleanup mismatched headers: "+err.Error()) // Non-fatal, continue } @@ -60,7 +60,7 @@ func setupKernelHeaders(log *Logger) error { if err := extractTarGz(headersTarball, headersDir); err != nil { return fmt.Errorf("extract headers: %w", err) } - log.Info("headers", "extracted kernel headers to "+headersDir) + log.Info("hypeman-init:headers", "extracted kernel headers to "+headersDir) // Create build symlink buildLink := filepath.Join(modulesDir, "build") @@ -70,7 +70,7 @@ func setupKernelHeaders(log *Logger) error { if err := os.Symlink(symlinkTarget, buildLink); err != nil { return fmt.Errorf("create build symlink: %w", err) } - log.Info("headers", "created build symlink") + log.Info("hypeman-init:headers", "created build symlink") return nil } @@ -91,7 +91,7 @@ func cleanupMismatchedModules(log *Logger, runningKernel string) error { } if entry.Name() != runningKernel { path := filepath.Join(newrootLibModules, entry.Name()) - log.Info("headers", "removing mismatched modules: "+entry.Name()) + log.Info("hypeman-init:headers", "removing mismatched modules: "+entry.Name()) if err := os.RemoveAll(path); err != nil { return fmt.Errorf("remove %s: %w", path, err) } @@ -120,7 +120,7 @@ func cleanupMismatchedHeaders(log *Logger, runningKernel string) error { // Remove any linux-headers-* directory that doesn't match if strings.HasPrefix(entry.Name(), "linux-headers-") && entry.Name() != expectedName { path := filepath.Join(newrootUsrSrc, entry.Name()) - log.Info("headers", "removing mismatched headers: "+entry.Name()) + log.Info("hypeman-init:headers", "removing mismatched headers: "+entry.Name()) if err := os.RemoveAll(path); err != nil { return fmt.Errorf("remove %s: %w", path, err) } diff --git a/lib/system/init/main.go b/lib/system/init/main.go index 43d22417..6eaa10cf 100644 --- a/lib/system/init/main.go +++ b/lib/system/init/main.go @@ -17,31 +17,31 @@ import ( func main() { log := NewLogger() - log.Info("boot", "init starting") + log.Info("hypeman-init:boot", "init starting") // Phase 1: Mount additional filesystems (proc/sys/dev already mounted by init.sh) if err := mountEssentials(log); err != nil { - log.Error("mount", "failed to mount essentials", err) + log.Error("hypeman-init:mount", "failed to mount essentials", err) dropToShell() } // Phase 2: Setup overlay rootfs if err := setupOverlay(log); err != nil { - log.Error("overlay", "failed to setup overlay", err) + log.Error("hypeman-init:overlay", "failed to setup overlay", err) dropToShell() } // Phase 3: Read and parse config cfg, err := readConfig(log) if err != nil { - log.Error("config", "failed to read config", err) + log.Error("hypeman-init:config", "failed to read config", err) dropToShell() } // Phase 4: Configure network (shared between modes) if cfg.NetworkEnabled { if err := configureNetwork(log, cfg); err != nil { - log.Error("network", "failed to configure network", err) + log.Error("hypeman-init:network", "failed to configure network", err) // Continue anyway - network isn't always required } } @@ -49,39 +49,39 @@ func main() { // Phase 5: Mount volumes if len(cfg.VolumeMounts) > 0 { if err := mountVolumes(log, cfg); err != nil { - log.Error("volumes", "failed to mount volumes", err) + log.Error("hypeman-init:volumes", "failed to mount volumes", err) // Continue anyway } } // Phase 6: Bind mount filesystems to new root if err := bindMountsToNewRoot(log); err != nil { - log.Error("bind", "failed to bind mounts", err) + log.Error("hypeman-init:bind", "failed to bind mounts", err) dropToShell() } // Phase 7: Copy guest-agent to target location (skips if already exists or skip_guest_agent=true) if err := copyGuestAgent(log, cfg.SkipGuestAgent); err != nil { - log.Error("agent", "failed to copy guest-agent", err) + log.Error("hypeman-init:agent", "failed to copy guest-agent", err) // Continue anyway - exec will still work, just no remote access } // Phase 8: Setup kernel headers for DKMS (can be skipped via config) if cfg.SkipKernelHeaders { - log.Info("headers", "skipping kernel headers setup (skip_kernel_headers=true)") + log.Info("hypeman-init:headers", "skipping kernel headers setup (skip_kernel_headers=true)") } else { if err := setupKernelHeaders(log); err != nil { - log.Error("headers", "failed to setup kernel headers", err) + log.Error("hypeman-init:headers", "failed to setup kernel headers", err) // Continue anyway - only needed for DKMS module building } } // Phase 9: Mode-specific execution if cfg.InitMode == "systemd" { - log.Info("mode", "entering systemd mode") + log.Info("hypeman-init:mode", "entering systemd mode") runSystemdMode(log, cfg) } else { - log.Info("mode", "entering exec mode") + log.Info("hypeman-init:mode", "entering exec mode") runExecMode(log, cfg) } } diff --git a/lib/system/init/mode_exec.go b/lib/system/init/mode_exec.go index e5744da6..dd253332 100644 --- a/lib/system/init/mode_exec.go +++ b/lib/system/init/mode_exec.go @@ -1,11 +1,14 @@ package main import ( + "bufio" "fmt" "os" "os/exec" + "os/signal" "strings" "syscall" + "time" "github.com/kernel/hypeman/lib/vmconfig" ) @@ -15,20 +18,20 @@ import ( // - The init binary remains PID 1 // - Guest-agent runs as a background process // - The container entrypoint runs as a child process -// - After entrypoint exits, guest-agent keeps VM alive +// - After entrypoint exits, init logs exit info and cleanly shuts down the VM func runExecMode(log *Logger, cfg *vmconfig.Config) { const newroot = "/overlay/newroot" // Change root to the new filesystem using chroot (consistent with systemd mode) - log.Info("exec", "executing chroot") + log.Info("hypeman-init:setup", "executing chroot") if err := syscall.Chroot(newroot); err != nil { - log.Error("exec", "chroot failed", err) + log.Error("hypeman-init:setup", "chroot failed", err) dropToShell() } // Change to new root directory if err := os.Chdir("/"); err != nil { - log.Error("exec", "chdir / failed", err) + log.Error("hypeman-init:setup", "chdir / failed", err) dropToShell() } @@ -40,15 +43,15 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) { // Pass environment variables so they're available via hypeman exec var agentCmd *exec.Cmd if cfg.SkipGuestAgent { - log.Info("exec", "skipping guest-agent (skip_guest_agent=true)") + log.Info("hypeman-init:setup", "skipping guest-agent (skip_guest_agent=true)") } else { - log.Info("exec", "starting guest-agent in background") + log.Info("hypeman-init:setup", "starting guest-agent in background") agentCmd = exec.Command("/opt/hypeman/guest-agent") agentCmd.Env = buildEnv(cfg.Env) agentCmd.Stdout = os.Stdout agentCmd.Stderr = os.Stderr if err := agentCmd.Start(); err != nil { - log.Error("exec", "failed to start guest-agent", err) + log.Error("hypeman-init:setup", "failed to start guest-agent", err) } } @@ -62,12 +65,12 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) { entrypoint := shellQuoteArgs(cfg.Entrypoint) cmd := shellQuoteArgs(cfg.Cmd) - log.Info("exec", fmt.Sprintf("workdir=%s entrypoint=%v cmd=%v", workdir, cfg.Entrypoint, cfg.Cmd)) + log.Info("hypeman-init:entrypoint", fmt.Sprintf("workdir=%s entrypoint=%v cmd=%v", workdir, cfg.Entrypoint, cfg.Cmd)) // Construct the shell command to run shellCmd := fmt.Sprintf("cd %s && exec %s %s", shellQuote(workdir), entrypoint, cmd) - log.Info("exec", "launching entrypoint") + log.Info("hypeman-init:entrypoint", "launching entrypoint") // Run the entrypoint without stdin (defaults to /dev/null). // This matches the old shell script behavior where the app ran in background with & @@ -81,32 +84,132 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) { appCmd.Env = buildEnv(cfg.Env) if err := appCmd.Start(); err != nil { - log.Error("exec", "failed to start entrypoint", err) + log.Error("hypeman-init:entrypoint", "failed to start entrypoint", err) dropToShell() } - log.Info("exec", fmt.Sprintf("container app started (PID %d)", appCmd.Process.Pid)) + log.Info("hypeman-init:entrypoint", fmt.Sprintf("container app started (PID %d)", appCmd.Process.Pid)) + + // Set up signal forwarding: when init receives a signal (e.g. from guest-agent + // Shutdown RPC), forward it to the entrypoint child process so it can gracefully + // shut down. This is how Docker/containerd works -- SIGTERM to PID 1 gets + // forwarded to the app. + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGINT) + go func() { + for sig := range sigCh { + if appCmd.Process != nil { + appCmd.Process.Signal(sig) + } + } + }() // Wait for app to exit err := appCmd.Wait() + signal.Stop(sigCh) + exitCode := 0 if err != nil { if exitErr, ok := err.(*exec.ExitError); ok { - exitCode = exitErr.ExitCode() + // Go's ExitCode() returns -1 when the process was killed by a signal. + // Check WaitStatus directly to get the signal and compute 128+signal + // (the standard shell convention for signal-killed processes). + if ws, ok := exitErr.Sys().(syscall.WaitStatus); ok && ws.Signaled() { + exitCode = 128 + int(ws.Signal()) + } else { + exitCode = exitErr.ExitCode() + } + } + } + + // Build human-readable exit description + exitMsg := describeExitCode(exitCode) + + // Log the exit with appropriate level + if exitCode == 0 { + log.Info("hypeman-init:entrypoint", "app exited with code 0 (success)") + } else { + log.Error("hypeman-init:entrypoint", fmt.Sprintf("app exited with code %d (%s)", exitCode, exitMsg), nil) + } + + // Write machine-parseable exit sentinel to serial console. + // The host reads this lazily from the serial console log file when it + // discovers the VM has stopped (socket gone -> Stopped state). + log.Info("hypeman-init:entrypoint", formatExitSentinel(exitCode, exitMsg)) + + // Clean shutdown: use reboot(POWER_OFF) instead of syscall.Exit to avoid + // kernel panic ("Attempted to kill init!"). This cleanly terminates the VM + // and causes the hypervisor process to exit on the host. + syscall.Sync() + syscall.Reboot(syscall.LINUX_REBOOT_CMD_POWER_OFF) +} + +// describeExitCode returns a human-readable description of an exit code. +func describeExitCode(code int) string { + switch { + case code == 0: + return "success" + case code == 126: + return "permission denied (command not executable)" + case code == 127: + return "command not found" + case code > 128: + sig := syscall.Signal(code - 128) + desc := fmt.Sprintf("killed by signal %d (%s)", code-128, sig.String()) + // Check for OOM on SIGKILL + if code == 137 { // 128 + 9 (SIGKILL) + if checkOOMKill() { + desc += " - OOM" + } } + return desc + default: + return fmt.Sprintf("exit code %d", code) } +} - log.Info("exec", fmt.Sprintf("app exited with code %d", exitCode)) +// formatExitSentinel returns a machine-parseable sentinel line for the host to parse. +// Format: HYPEMAN-EXIT code= message="" +func formatExitSentinel(code int, message string) string { + return fmt.Sprintf("HYPEMAN-EXIT code=%d message=%q", code, message) +} - // Wait for guest-agent (keeps init alive, prevents kernel panic) - // The guest-agent runs forever, so this effectively keeps the VM alive - // until it's explicitly terminated - if agentCmd != nil && agentCmd.Process != nil { - agentCmd.Wait() +// checkOOMKill checks /dev/kmsg for recent OOM kill messages. +// Returns true if an OOM kill was detected. +// Uses a 1s timeout to avoid hanging if /dev/kmsg blocks at end of buffer. +func checkOOMKill() bool { + f, err := os.OpenFile("/dev/kmsg", os.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return false + } + defer f.Close() + + // Use a goroutine with timeout since /dev/kmsg can still block in some cases + result := make(chan bool, 1) + go func() { + scanner := bufio.NewScanner(f) + for scanner.Scan() { + if isOOMLine(scanner.Text()) { + result <- true + return + } + } + result <- false + }() + + select { + case found := <-result: + return found + case <-time.After(1 * time.Second): + return false } +} - // Exit with the app's exit code - syscall.Exit(exitCode) +// isOOMLine returns true if a kernel log line indicates an OOM kill event. +func isOOMLine(line string) bool { + return strings.Contains(line, "Out of memory") || + strings.Contains(line, "oom-kill") || + strings.Contains(line, "oom_reaper") } // buildEnv constructs environment variables from the config. diff --git a/lib/system/init/mode_exec_test.go b/lib/system/init/mode_exec_test.go new file mode 100644 index 00000000..0255f22d --- /dev/null +++ b/lib/system/init/mode_exec_test.go @@ -0,0 +1,156 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDescribeExitCode(t *testing.T) { + tests := []struct { + name string + code int + contains string // substring that must appear + }{ + { + name: "success", + code: 0, + contains: "success", + }, + { + name: "generic exit code", + code: 1, + contains: "exit code 1", + }, + { + name: "permission denied", + code: 126, + contains: "permission denied", + }, + { + name: "command not found", + code: 127, + contains: "command not found", + }, + { + name: "SIGKILL (137)", + code: 137, + contains: "signal 9", + }, + { + name: "SIGTERM (143)", + code: 143, + contains: "signal 15", + }, + { + name: "SIGSEGV (139)", + code: 139, + contains: "signal 11", + }, + { + name: "SIGHUP (129)", + code: 129, + contains: "signal 1", + }, + { + name: "generic non-zero", + code: 42, + contains: "exit code 42", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := describeExitCode(tc.code) + assert.Contains(t, result, tc.contains, + "describeExitCode(%d) = %q should contain %q", tc.code, result, tc.contains) + }) + } +} + +func TestFormatExitSentinel(t *testing.T) { + tests := []struct { + name string + code int + message string + want string + }{ + { + name: "success", + code: 0, + message: "success", + want: `HYPEMAN-EXIT code=0 message="success"`, + }, + { + name: "command not found", + code: 127, + message: "command not found", + want: `HYPEMAN-EXIT code=127 message="command not found"`, + }, + { + name: "SIGKILL", + code: 137, + message: `killed by signal 9 (killed)`, + want: `HYPEMAN-EXIT code=137 message="killed by signal 9 (killed)"`, + }, + { + name: "message with quotes", + code: 1, + message: `error: "bad thing"`, + want: `HYPEMAN-EXIT code=1 message="error: \"bad thing\""`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := formatExitSentinel(tc.code, tc.message) + require.Equal(t, tc.want, result) + }) + } +} + +func TestIsOOMLine(t *testing.T) { + tests := []struct { + name string + line string + want bool + }{ + { + name: "Out of memory message", + line: "6,1234,56789,-;Out of memory: Killed process 42 (my-app) total-vm:1024kB", + want: true, + }, + { + name: "oom-kill event", + line: "6,1235,56790,-;oom-kill:constraint=CONSTRAINT_MEMCG,nodemask=(null),cpuset=/,mems_allowed=0", + want: true, + }, + { + name: "oom_reaper message", + line: "6,1236,56791,-;oom_reaper: reaped process 1234 (my-app), now anon-rss:0kB", + want: true, + }, + { + name: "normal kernel message", + line: "6,1237,56792,-;eth0: link up, 1000Mbps, full-duplex", + want: false, + }, + { + name: "empty line", + line: "", + want: false, + }, + { + name: "process killed by user (not OOM)", + line: "6,1238,56793,-;process 42 exited with signal 9", + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isOOMLine(tc.line)) + }) + } +} diff --git a/lib/system/init/mode_systemd.go b/lib/system/init/mode_systemd.go index fb00fc79..50fc0bbd 100644 --- a/lib/system/init/mode_systemd.go +++ b/lib/system/init/mode_systemd.go @@ -21,25 +21,25 @@ func runSystemdMode(log *Logger, cfg *vmconfig.Config) { // Inject hypeman-agent.service (skip if guest-agent was not copied) // Pass environment variables so they're available via hypeman exec if cfg.SkipGuestAgent { - log.Info("systemd", "skipping agent service injection (skip_guest_agent=true)") + log.Info("hypeman-init:systemd", "skipping agent service injection (skip_guest_agent=true)") } else { - log.Info("systemd", "injecting hypeman-agent.service") + log.Info("hypeman-init:systemd", "injecting hypeman-agent.service") if err := injectAgentService(newroot, cfg.Env); err != nil { - log.Error("systemd", "failed to inject service", err) + log.Error("hypeman-init:systemd", "failed to inject service", err) // Continue anyway - VM will work, just without agent } } // Change root to the new filesystem using chroot - log.Info("systemd", "executing chroot") + log.Info("hypeman-init:systemd", "executing chroot") if err := syscall.Chroot(newroot); err != nil { - log.Error("systemd", "chroot failed", err) + log.Error("hypeman-init:systemd", "chroot failed", err) dropToShell() } // Change to new root directory if err := os.Chdir("/"); err != nil { - log.Error("systemd", "chdir / failed", err) + log.Error("hypeman-init:systemd", "chdir / failed", err) dropToShell() } @@ -51,13 +51,13 @@ func runSystemdMode(log *Logger, cfg *vmconfig.Config) { } // Exec systemd - this replaces the current process - log.Info("systemd", fmt.Sprintf("exec %v", argv)) + log.Info("hypeman-init:systemd", fmt.Sprintf("exec %v", argv)) // syscall.Exec replaces the current process with the new one // Use buildEnv to include user's environment variables from the image/instance config err := syscall.Exec(argv[0], argv, buildEnv(cfg.Env)) if err != nil { - log.Error("systemd", fmt.Sprintf("exec %s failed", argv[0]), err) + log.Error("hypeman-init:systemd", fmt.Sprintf("exec %s failed", argv[0]), err) dropToShell() } } diff --git a/lib/system/init/mount.go b/lib/system/init/mount.go index 07894d01..9259cc77 100644 --- a/lib/system/init/mount.go +++ b/lib/system/init/mount.go @@ -41,12 +41,12 @@ func mountEssentials(log *Logger) error { } if err := syscall.Mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, ""); err != nil { // Non-fatal: some kernels may not have cgroup2 support - log.Info("mount", "cgroup2 mount failed (non-fatal): "+err.Error()) + log.Info("hypeman-init:mount", "cgroup2 mount failed (non-fatal): "+err.Error()) } else { - log.Info("mount", "mounted cgroup2") + log.Info("hypeman-init:mount", "mounted cgroup2") } - log.Info("mount", "mounted devpts/shm") + log.Info("hypeman-init:mount", "mounted devpts/shm") // Set up serial console now that /dev is mounted // hvc0 for Virtualization.framework (vz) on macOS @@ -57,12 +57,12 @@ func mountEssentials(log *Logger) error { if _, err := os.Stat(console); err == nil { log.SetConsole(console) redirectToConsole(console) - log.Info("mount", "using console "+console) + log.Info("hypeman-init:mount", "using console "+console) break } } - log.Info("mount", "console setup complete") + log.Info("hypeman-init:mount", "console setup complete") return nil } @@ -109,7 +109,7 @@ func setupOverlay(log *Logger) error { if err := mount("/dev/vda", "/lower", "ext4", "ro"); err != nil { return fmt.Errorf("mount rootfs: %w", err) } - log.Info("overlay", "mounted rootfs from /dev/vda") + log.Info("hypeman-init:overlay", "mounted rootfs from /dev/vda") // Mount writable overlay disk from /dev/vdb if err := mount("/dev/vdb", "/overlay", "ext4", ""); err != nil { @@ -122,13 +122,13 @@ func setupOverlay(log *Logger) error { return fmt.Errorf("mkdir %s: %w", dir, err) } } - log.Info("overlay", "mounted overlay disk from /dev/vdb") + log.Info("hypeman-init:overlay", "mounted overlay disk from /dev/vdb") // Create overlay filesystem if err := mountOverlay("/lower", "/overlay/upper", "/overlay/work", "/overlay/newroot"); err != nil { return fmt.Errorf("mount overlay: %w", err) } - log.Info("overlay", "created overlay filesystem") + log.Info("hypeman-init:overlay", "created overlay filesystem") return nil } @@ -161,7 +161,7 @@ func bindMountsToNewRoot(log *Logger) error { } } - log.Info("bind", "bound mounts to new root") + log.Info("hypeman-init:bind", "bound mounts to new root") // Set up /dev symlinks for process substitution inside the container symlinks := []struct{ target, link string }{ @@ -235,13 +235,13 @@ func copyGuestAgent(log *Logger, skipGuestAgent bool) error { // Check for skip via config if skipGuestAgent { - log.Info("agent", "skipping guest-agent copy (skip_guest_agent=true)") + log.Info("hypeman-init:agent", "skipping guest-agent copy (skip_guest_agent=true)") return nil } // Check if destination already exists (lazy copy - skip if already present) if _, err := os.Stat(dst); err == nil { - log.Info("agent", "guest-agent already exists, skipping copy") + log.Info("hypeman-init:agent", "guest-agent already exists, skipping copy") return nil } @@ -261,7 +261,7 @@ func copyGuestAgent(log *Logger, skipGuestAgent bool) error { return fmt.Errorf("write destination: %w", err) } - log.Info("agent", "copied guest-agent to /opt/hypeman/") + log.Info("hypeman-init:agent", "copied guest-agent to /opt/hypeman/") return nil } diff --git a/lib/system/init/network.go b/lib/system/init/network.go index cb5f6941..01c056d6 100644 --- a/lib/system/init/network.go +++ b/lib/system/init/network.go @@ -45,7 +45,7 @@ func configureNetwork(log *Logger, cfg *vmconfig.Config) error { return fmt.Errorf("write resolv.conf: %w", err) } - log.Info("network", fmt.Sprintf("configured eth0 with %s", addr)) + log.Info("hypeman-init:network", fmt.Sprintf("configured eth0 with %s", addr)) return nil } diff --git a/lib/system/init/volumes.go b/lib/system/init/volumes.go index 459c1042..5f24572b 100644 --- a/lib/system/init/volumes.go +++ b/lib/system/init/volumes.go @@ -12,29 +12,29 @@ import ( // mountVolumes mounts attached volumes according to the configuration. // Supports three modes: ro (read-only), rw (read-write), and overlay. func mountVolumes(log *Logger, cfg *vmconfig.Config) error { - log.Info("volumes", "mounting volumes") + log.Info("hypeman-init:volumes", "mounting volumes") for _, vol := range cfg.VolumeMounts { mountPath := filepath.Join("/overlay/newroot", vol.Path) // Create mount point if err := os.MkdirAll(mountPath, 0755); err != nil { - log.Error("volumes", fmt.Sprintf("mkdir %s failed", vol.Path), err) + log.Error("hypeman-init:volumes", fmt.Sprintf("mkdir %s failed", vol.Path), err) continue } switch vol.Mode { case "overlay": if err := mountVolumeOverlay(log, vol, mountPath); err != nil { - log.Error("volumes", fmt.Sprintf("mount overlay %s failed", vol.Path), err) + log.Error("hypeman-init:volumes", fmt.Sprintf("mount overlay %s failed", vol.Path), err) } case "ro": if err := mountVolumeReadOnly(log, vol, mountPath); err != nil { - log.Error("volumes", fmt.Sprintf("mount ro %s failed", vol.Path), err) + log.Error("hypeman-init:volumes", fmt.Sprintf("mount ro %s failed", vol.Path), err) } default: // "rw" if err := mountVolumeReadWrite(log, vol, mountPath); err != nil { - log.Error("volumes", fmt.Sprintf("mount rw %s failed", vol.Path), err) + log.Error("hypeman-init:volumes", fmt.Sprintf("mount rw %s failed", vol.Path), err) } } } @@ -84,7 +84,7 @@ func mountVolumeOverlay(log *Logger, vol vmconfig.VolumeMount, mountPath string) return fmt.Errorf("mount overlay: %s: %s", err, output) } - log.Info("volumes", fmt.Sprintf("mounted %s at %s (overlay via %s)", vol.Device, vol.Path, vol.OverlayDevice)) + log.Info("hypeman-init:volumes", fmt.Sprintf("mounted %s at %s (overlay via %s)", vol.Device, vol.Path, vol.OverlayDevice)) return nil } @@ -96,7 +96,7 @@ func mountVolumeReadOnly(log *Logger, vol vmconfig.VolumeMount, mountPath string return fmt.Errorf("%s: %s", err, output) } - log.Info("volumes", fmt.Sprintf("mounted %s at %s (ro)", vol.Device, vol.Path)) + log.Info("hypeman-init:volumes", fmt.Sprintf("mounted %s at %s (ro)", vol.Device, vol.Path)) return nil } @@ -107,7 +107,7 @@ func mountVolumeReadWrite(log *Logger, vol vmconfig.VolumeMount, mountPath strin return fmt.Errorf("%s: %s", err, output) } - log.Info("volumes", fmt.Sprintf("mounted %s at %s (rw)", vol.Device, vol.Path)) + log.Info("hypeman-init:volumes", fmt.Sprintf("mounted %s at %s (rw)", vol.Device, vol.Path)) return nil } diff --git a/openapi.yaml b/openapi.yaml index b89dec5b..ca918ede 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -204,6 +204,18 @@ components: The instance will still run, but remote command execution will be unavailable. default: false example: false + entrypoint: + type: array + items: + type: string + description: Override image entrypoint (like docker run --entrypoint). Omit to use image default. + example: ["/bin/sh", "-c"] + cmd: + type: array + items: + type: string + description: Override image CMD (like docker run ). Omit to use image default. + example: ["echo", "hello"] # Future: port_mappings, timeout_seconds Instance: @@ -313,6 +325,15 @@ components: description: Stop timestamp (RFC3339) example: "2025-01-15T12:30:00Z" nullable: true + exit_code: + type: integer + description: App exit code (null if VM hasn't exited) + nullable: true + example: 137 + exit_message: + type: string + description: Human-readable description of exit (e.g., "command not found", "killed by signal 9 (SIGKILL) - OOM") + example: "killed by signal 9 (SIGKILL)" has_snapshot: type: boolean description: Whether a snapshot exists for this instance @@ -1527,6 +1548,23 @@ paths: schema: type: string description: Instance ID or name + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + entrypoint: + type: array + items: + type: string + description: Override image entrypoint for this run. Omit to keep previous value. + cmd: + type: array + items: + type: string + description: Override image CMD for this run. Omit to keep previous value. responses: 200: description: Instance started