From 1c577bc710f6dd63efcee6658d260b190cfe5c2d Mon Sep 17 00:00:00 2001 From: gabriel-farache Date: Wed, 12 Aug 2026 15:50:26 +0200 Subject: [PATCH 1/9] feat(agent): add environment agent support, remove direct SP management Introduce agent-based resource provisioning via NATS and remove all direct Service Provider (SP) management, completing the transition to an agent-only architecture. Each resource in a run is now routed to an agent (policy.SelectedAgent) rather than a provider, with per-resource SPRM provisioning using AgentName. - Agent domain (internal/agent/): store, service, HTTP handler (split into handler/convert/errors, matching the SP pattern), health monitor, OpenAPI spec and generated server/client/types - NATS integration (internal/sp/messaging/, internal/sp/consumer/, internal/sp/pending/): CloudEvents publish/consume and sweep-driven pending/queued retries and self-heal, including a CAS fix for a race that let two concurrent healers reassign the same instance to different agents, and proactive reassignment of run-siblings stuck on an excluded agent - Policy: ServiceTypes and Cost threaded into agent evaluation so Rego can see agent capability, with a hard capability pre-filter and explicit rejection when no capable or all-excluded agents remain (previously silent/fail-open) - Placement: agent-aware routing in CreateRun/RehydrateResource/ ReEvaluateWithExclude; ProviderName replaced by AgentName across the store model, DTO, and ListRun filter; fail closed instead of open when listing ready agents errors - SP resource manager: CreateInstance publishes to NATS instead of calling provider HTTP, with AgentName on ServiceTypeInstance and a matching agent_name filter on ListInstances - Removed all provider-specific code (internal/sp/*/provider, api/sp/*/provider, pkg/sp/client/provider) - Subsystem coverage for NATS response events, self-heal sweep re-routing, and sibling reassignment during self-heal (PR #37 review thread r3761347505) Assisted by: Cursor - Sonnet 5 Co-authored-by: Cursor Signed-off-by: gabriel-farache --- .gitignore | 2 + Makefile | 1 + api/agent/v1alpha1/openapi.yaml | 374 +++++ .../provider => agent/v1alpha1}/spec.gen.cfg | 2 +- api/agent/v1alpha1/spec.gen.go | 148 ++ .../provider => agent/v1alpha1}/types.gen.cfg | 2 +- api/agent/v1alpha1/types.gen.go | 230 +++ api/sp/v1alpha1/provider/openapi.yaml | 470 ------ api/sp/v1alpha1/provider/spec.gen.go | 158 -- api/sp/v1alpha1/provider/types.gen.go | 258 ---- api/sp/v1alpha1/resource_manager/openapi.yaml | 39 +- api/sp/v1alpha1/resource_manager/spec.gen.go | 87 +- api/sp/v1alpha1/resource_manager/types.gen.go | 24 +- go.mod | 3 +- go.sum | 2 - .../api/server}/server.gen.cfg | 2 +- internal/agent/api/server/server.gen.go | 1000 ++++++++++++ internal/agent/handlers/v1alpha1/convert.go | 40 + internal/agent/handlers/v1alpha1/errors.go | 94 ++ .../agent/handlers/v1alpha1/errors_test.go | 112 ++ internal/agent/handlers/v1alpha1/handler.go | 93 ++ .../handlers/v1alpha1/handler_suite_test.go} | 6 +- .../agent/handlers/v1alpha1/handler_test.go | 214 +++ .../healthcheck/healthcheck_suite_test.go | 4 +- internal/agent/healthcheck/monitor.go | 67 + internal/agent/healthcheck/monitor_test.go | 144 ++ internal/agent/service/agent.go | 174 +++ internal/agent/service/agent_suite_test.go | 13 + internal/agent/service/agent_test.go | 253 +++ internal/agent/service/errors.go | 59 + internal/agent/store/agent/agent.go | 251 +++ .../store/agent/agent_suite_test.go} | 6 +- internal/agent/store/agent/agent_test.go | 398 +++++ internal/agent/store/agent/pagination.go | 42 + internal/agent/store/model/agent.go | 42 + internal/app/config.go | 12 + internal/app/db.go | 3 +- internal/app/openapi.go | 37 +- internal/app/openapi_validation_test.go | 6 - internal/app/run.go | 145 +- internal/catalog/placement/local_client.go | 18 +- .../catalog/service/catalog_item_instance.go | 5 +- .../service/catalog_item_instance_test.go | 45 + internal/placement/agent/service_client.go | 39 + .../placement/agent/service_client_test.go | 63 + internal/placement/agent/types.go | 20 + internal/placement/policy/service_client.go | 13 +- .../placement/policy/service_client_test.go | 155 ++ internal/placement/policy/types.go | 21 +- internal/placement/service/convert.go | 2 +- internal/placement/service/errors.go | 33 +- internal/placement/service/placement.go | 251 ++- internal/placement/service/placement_test.go | 589 ++++++- internal/placement/sprm/service_client.go | 32 +- .../placement/sprm/service_client_test.go | 89 ++ internal/placement/sprm/types.go | 16 +- internal/placement/store/model/resource.go | 10 +- internal/placement/store/resource.go | 24 +- internal/placement/store/resource_test.go | 112 +- internal/placement/types/resource.go | 2 +- internal/policy/opa/evaluation.go | 54 +- internal/policy/opa/evaluation_test.go | 22 +- .../policy/service/agent_constraints_test.go | 113 ++ internal/policy/service/constraints.go | 219 +-- internal/policy/service/constraints_test.go | 80 - internal/policy/service/errors.go | 31 +- internal/policy/service/evaluation.go | 229 ++- internal/policy/service/evaluation_test.go | 480 +++++- internal/sp/api/provider/server.gen.go | 1354 ----------------- .../sp/api/resource_manager/server.gen.go | 68 +- internal/sp/cleanup/scheduler.go | 126 +- internal/sp/cleanup/scheduler_test.go | 179 ++- internal/sp/config/config.go | 17 +- internal/sp/consumer/consumer.go | 11 +- internal/sp/consumer/consumer_test.go | 41 +- internal/sp/consumer/response_consumer.go | 366 +++++ .../sp/consumer/response_consumer_test.go | 637 ++++++++ internal/sp/handlers/provider/handler.go | 170 --- internal/sp/handlers/provider/handler_test.go | 284 ---- .../sp/handlers/resource_manager/convert.go | 19 +- .../sp/handlers/resource_manager/errors.go | 27 +- .../handlers/resource_manager/errors_test.go | 29 + .../sp/handlers/resource_manager/handler.go | 10 +- .../handlers/resource_manager/handler_test.go | 252 +-- internal/sp/healthcheck/monitor.go | 216 --- internal/sp/healthcheck/monitor_test.go | 601 -------- .../messaging_suite_test.go} | 6 +- internal/sp/messaging/publisher.go | 94 ++ internal/sp/messaging/publisher_test.go | 287 ++++ internal/sp/messaging/types.go | 52 + internal/sp/pending/pending_suite_test.go | 13 + internal/sp/pending/sweep.go | 353 +++++ internal/sp/pending/sweep_test.go | 513 +++++++ internal/sp/service/errors.go | 24 +- internal/sp/service/provider/convert.go | 91 -- internal/sp/service/provider/provider.go | 316 ---- internal/sp/service/provider/provider_test.go | 455 ------ .../sp/service/resource_manager/convert.go | 20 +- .../resource_manager/service_type_instance.go | 407 ++--- .../service_type_instance_test.go | 825 +++++----- internal/sp/store/db.go | 2 +- internal/sp/store/model/provider.go | 43 - .../sp/store/model/service_type_instance.go | 28 +- internal/sp/store/model/status.go | 21 + internal/sp/store/provider/provider.go | 196 --- internal/sp/store/provider/provider_test.go | 449 ------ .../resource_manager/service_instance.go | 262 +++- .../resource_manager/service_instance_test.go | 625 +++++--- internal/sp/store/store.go | 8 - internal/sp/store/store_test.go | 9 - make/agent.mk | 37 + make/sp.mk | 41 +- pkg/agent/client/client.gen.cfg | 8 + pkg/agent/client/client.gen.go | 811 ++++++++++ pkg/sp/client/provider/client.gen.cfg | 9 - pkg/sp/client/provider/client.gen.go | 1067 ------------- pkg/sp/client/resource_manager/client.gen.go | 24 +- test/subsystem/sp/docker-compose.yaml | 10 + test/subsystem/sp/provider_test.go | 151 -- test/subsystem/sp/response_events_test.go | 95 ++ test/subsystem/sp/self_heal_test.go | 295 ++++ test/subsystem/sp/service_instance_test.go | 412 +---- test/subsystem/sp/setup_test.go | 472 ++++-- test/subsystem/sp/suite_test.go | 61 + .../sp/testdata/agent_selecting_policy.rego | 8 + .../three_agent_selecting_policy.rego | 27 + .../testdata/two_agent_selecting_policy.rego | 19 + 127 files changed, 12106 insertions(+), 8731 deletions(-) create mode 100644 api/agent/v1alpha1/openapi.yaml rename api/{sp/v1alpha1/provider => agent/v1alpha1}/spec.gen.cfg (78%) create mode 100644 api/agent/v1alpha1/spec.gen.go rename api/{sp/v1alpha1/provider => agent/v1alpha1}/types.gen.cfg (76%) create mode 100644 api/agent/v1alpha1/types.gen.go delete mode 100644 api/sp/v1alpha1/provider/openapi.yaml delete mode 100644 api/sp/v1alpha1/provider/spec.gen.go delete mode 100644 api/sp/v1alpha1/provider/types.gen.go rename internal/{sp/api/provider => agent/api/server}/server.gen.cfg (84%) create mode 100644 internal/agent/api/server/server.gen.go create mode 100644 internal/agent/handlers/v1alpha1/convert.go create mode 100644 internal/agent/handlers/v1alpha1/errors.go create mode 100644 internal/agent/handlers/v1alpha1/errors_test.go create mode 100644 internal/agent/handlers/v1alpha1/handler.go rename internal/{sp/service/provider/provider_suite_test.go => agent/handlers/v1alpha1/handler_suite_test.go} (52%) create mode 100644 internal/agent/handlers/v1alpha1/handler_test.go rename internal/{sp => agent}/healthcheck/healthcheck_suite_test.go (63%) create mode 100644 internal/agent/healthcheck/monitor.go create mode 100644 internal/agent/healthcheck/monitor_test.go create mode 100644 internal/agent/service/agent.go create mode 100644 internal/agent/service/agent_suite_test.go create mode 100644 internal/agent/service/agent_test.go create mode 100644 internal/agent/service/errors.go create mode 100644 internal/agent/store/agent/agent.go rename internal/{sp/handlers/provider/handler_suite_test.go => agent/store/agent/agent_suite_test.go} (55%) create mode 100644 internal/agent/store/agent/agent_test.go create mode 100644 internal/agent/store/agent/pagination.go create mode 100644 internal/agent/store/model/agent.go create mode 100644 internal/placement/agent/service_client.go create mode 100644 internal/placement/agent/service_client_test.go create mode 100644 internal/placement/agent/types.go create mode 100644 internal/placement/policy/service_client_test.go create mode 100644 internal/placement/sprm/service_client_test.go create mode 100644 internal/policy/service/agent_constraints_test.go delete mode 100644 internal/sp/api/provider/server.gen.go create mode 100644 internal/sp/consumer/response_consumer.go create mode 100644 internal/sp/consumer/response_consumer_test.go delete mode 100644 internal/sp/handlers/provider/handler.go delete mode 100644 internal/sp/handlers/provider/handler_test.go create mode 100644 internal/sp/handlers/resource_manager/errors_test.go delete mode 100644 internal/sp/healthcheck/monitor.go delete mode 100644 internal/sp/healthcheck/monitor_test.go rename internal/sp/{store/provider/provider_suite_test.go => messaging/messaging_suite_test.go} (54%) create mode 100644 internal/sp/messaging/publisher.go create mode 100644 internal/sp/messaging/publisher_test.go create mode 100644 internal/sp/messaging/types.go create mode 100644 internal/sp/pending/pending_suite_test.go create mode 100644 internal/sp/pending/sweep.go create mode 100644 internal/sp/pending/sweep_test.go delete mode 100644 internal/sp/service/provider/convert.go delete mode 100644 internal/sp/service/provider/provider.go delete mode 100644 internal/sp/service/provider/provider_test.go delete mode 100644 internal/sp/store/model/provider.go create mode 100644 internal/sp/store/model/status.go delete mode 100644 internal/sp/store/provider/provider.go delete mode 100644 internal/sp/store/provider/provider_test.go create mode 100644 make/agent.mk create mode 100644 pkg/agent/client/client.gen.cfg create mode 100644 pkg/agent/client/client.gen.go delete mode 100644 pkg/sp/client/provider/client.gen.cfg delete mode 100644 pkg/sp/client/provider/client.gen.go delete mode 100644 test/subsystem/sp/provider_test.go create mode 100644 test/subsystem/sp/response_events_test.go create mode 100644 test/subsystem/sp/self_heal_test.go create mode 100644 test/subsystem/sp/testdata/agent_selecting_policy.rego create mode 100644 test/subsystem/sp/testdata/three_agent_selecting_policy.rego create mode 100644 test/subsystem/sp/testdata/two_agent_selecting_policy.rego diff --git a/.gitignore b/.gitignore index 5420e38..b871ece 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ go.work.sum # Editor/IDE # .vscode/ /.idea/ + +.ai/ \ No newline at end of file diff --git a/Makefile b/Makefile index 1f3de9f..acaa939 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,7 @@ include make/catalog.mk include make/placement.mk include make/policy.mk include make/sp.mk +include make/agent.mk # Same as Containerfile: static build, no CGO (Postgres in prod/compose). # For SQLite local dev use make run (go run with CGO). diff --git a/api/agent/v1alpha1/openapi.yaml b/api/agent/v1alpha1/openapi.yaml new file mode 100644 index 0000000..a731d1b --- /dev/null +++ b/api/agent/v1alpha1/openapi.yaml @@ -0,0 +1,374 @@ +openapi: 3.0.4 +info: + contact: {} + description: | + DCM Agent API - Registration and management of Environment Agents. + Agents register with the control plane and send periodic heartbeats. + title: Agent API + version: v1alpha1 + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html + +servers: + - url: /api/v1alpha1 + +security: + - bearerAuth: [] + +tags: + - name: agent + description: Agent management operations + +paths: + /agents: + get: + tags: + - agent + summary: List all agents + operationId: listAgents + description: Returns a list of registered agents with optional filtering + parameters: + - name: health_status + in: query + description: Filter agents by health status + schema: + type: string + enum: + - ready + - congested + - unavailable + - name: max_page_size + in: query + description: Maximum number of results per page + schema: + type: integer + minimum: 1 + maximum: 100 + default: 100 + - name: page_token + in: query + description: Token for pagination + schema: + type: string + responses: + '200': + description: Successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/AgentList' + '400': + description: Bad request + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Error' + default: + description: Unexpected error + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Error' + + post: + tags: + - agent + summary: Register an agent + operationId: createAgent + description: | + Register a new agent or update an existing one. + Registration is idempotent by name: + - If name does not exist, a new agent entry is created (201) + - If name exists, the entry is updated and heartbeat is reset (200) + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/AgentRegistrationRequest' + responses: + '200': + description: Agent updated (re-registration) + content: + application/json: + schema: + $ref: '#/components/schemas/Agent' + '201': + description: Agent created (new registration) + content: + application/json: + schema: + $ref: '#/components/schemas/Agent' + '400': + description: Invalid input + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Error' + '409': + description: Agent name is already registered with a different topic_name, or vice versa + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Error' + default: + description: Unexpected error + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Error' + + /agents/{agentId}: + get: + tags: + - agent + summary: Get an agent + operationId: getAgent + description: Get an agent by its unique ID + parameters: + - $ref: '#/components/parameters/AgentIdPath' + responses: + '200': + description: Successful operation + content: + application/json: + schema: + $ref: '#/components/schemas/Agent' + '400': + description: Invalid ID supplied + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Error' + '404': + description: Agent not found + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Error' + default: + description: Unexpected error + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Error' + + /agents/{agentId}/heartbeat: + put: + tags: + - agent + summary: Send agent heartbeat + operationId: agentHeartbeat + description: | + Agents send periodic heartbeats to report health and consumer lag. + Heartbeat timestamps are monotonic: stale timestamps are ignored. + parameters: + - $ref: '#/components/parameters/AgentIdPath' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/HeartbeatRequest' + responses: + '200': + description: Heartbeat accepted + content: + application/json: + schema: + $ref: '#/components/schemas/Agent' + '400': + description: Invalid input + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Error' + '404': + description: Agent not found + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Error' + default: + description: Unexpected error + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Error' + +components: + parameters: + AgentIdPath: + name: agentId + in: path + required: true + schema: + type: string + minLength: 1 + maxLength: 63 + description: Unique identifier of the agent + + schemas: + AgentRegistrationRequest: + type: object + description: Request body for agent registration + required: + - name + - environment + - topic_name + - service_types + - cost + properties: + name: + type: string + description: Unique name of the agent + example: "env-agent-west-1" + environment: + type: string + description: Environment label for the agent + example: "production" + service_types: + type: array + items: + type: string + description: List of service types this agent can provide + example: ["vm", "container"] + cost: + type: string + enum: + - low + - medium-low + - medium + - medium-high + - high + description: Relative cost weight for placement decisions + example: "medium" + topic_name: + type: string + description: NATS topic name for this agent (must start with dcm.agent.) + pattern: '^dcm\.agent\..+' + example: "dcm.agent.env-agent-west-1" + + Agent: + type: object + description: Full agent resource representation + properties: + agent_id: + type: string + readOnly: true + description: Server-generated unique identifier + example: "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + name: + type: string + description: Unique name of the agent + example: "env-agent-west-1" + environment: + type: string + description: Environment label for the agent + example: "production" + service_types: + type: array + items: + type: string + description: List of service types this agent can provide + example: ["vm", "container"] + cost: + type: string + enum: + - low + - medium-low + - medium + - medium-high + - high + description: Relative cost weight for placement decisions + example: "medium" + topic_name: + type: string + description: NATS topic name for this agent + example: "dcm.agent.env-agent-west-1" + health_status: + type: string + description: Current health status of the agent + readOnly: true + enum: + - ready + - congested + - unavailable + example: "ready" + last_heartbeat: + type: string + format: date-time + readOnly: true + description: Timestamp of last heartbeat received + create_time: + type: string + format: date-time + readOnly: true + description: Timestamp when the agent was first registered + update_time: + type: string + format: date-time + readOnly: true + description: Timestamp when the agent was last updated + + HeartbeatRequest: + type: object + description: Request body for agent heartbeat + required: + - consumer_lag + - timestamp + properties: + consumer_lag: + type: integer + format: int64 + description: Number of unprocessed messages in the agent's NATS consumer + example: 0 + timestamp: + type: string + format: date-time + description: Timestamp of this heartbeat (used for monotonicity check) + + AgentList: + type: object + description: Paginated list of agents + properties: + agents: + type: array + items: + $ref: '#/components/schemas/Agent' + next_page_token: + type: string + description: Token for retrieving the next page of results + + Error: + type: object + description: RFC 7807 compliant error response + required: + - type + - title + properties: + type: + type: string + format: uri-reference + description: URI reference identifying the error type + title: + type: string + description: Short human-readable summary of the problem + status: + type: integer + description: HTTP status code + detail: + type: string + description: Human-readable explanation specific to this occurrence + instance: + type: string + format: uri-reference + description: URI reference for this specific error occurrence + + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT + description: JWT token obtained from the configured Auth Provider diff --git a/api/sp/v1alpha1/provider/spec.gen.cfg b/api/agent/v1alpha1/spec.gen.cfg similarity index 78% rename from api/sp/v1alpha1/provider/spec.gen.cfg rename to api/agent/v1alpha1/spec.gen.cfg index 3e86cec..77b7fd6 100644 --- a/api/sp/v1alpha1/provider/spec.gen.cfg +++ b/api/agent/v1alpha1/spec.gen.cfg @@ -1,4 +1,4 @@ -package: provider +package: agent generate: embedded-spec: true output-options: diff --git a/api/agent/v1alpha1/spec.gen.go b/api/agent/v1alpha1/spec.gen.go new file mode 100644 index 0000000..4cebcee --- /dev/null +++ b/api/agent/v1alpha1/spec.gen.go @@ -0,0 +1,148 @@ +// Package agent provides primitives to interact with the openapi HTTP API. +// +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.7.0 DO NOT EDIT. +package agent + +import ( + "bytes" + "compress/flate" + "encoding/base64" + "fmt" + "net/url" + "path" + "strings" + + "github.com/getkin/kin-openapi/openapi3" +) + +// Base64 encoded, compressed with deflate, json marshaled OpenAPI spec. +// Stored as a slice of fixed-width chunks rather than one concatenated +// const string: with thousands of chunks the chained `+` fold is several +// times slower for the Go compiler than parsing a slice literal. +var swaggerSpec = []string{ + "7Fltc9s2Ev4rO7ibuXhKvdhxk1bfXCdp3El7ntidfrB9HghcUmhJAAFAybqM/vvNAqRIiXTi9MWXzuST", + "TRLAvj/7LPSeCV0arVB5x2bvmeGWl+jRhqeTHJU/S8+5X9Bjik5YabzUis3Yz0q+qxBkisrLTKIFnYFf", + "IHDaxRImaZWhvQlTvEQ2YzweyBJm8V0lLaZs5m2FCXNigSUnKSW/e4MqJ5HPniaslKp5PEyYXxs6x3kr", + "Vc42m02zs9W3r+mrqiiiVmDR6coKBIvGokPleViUMGO1QeslhpPC6luZ9g+7QLtEO8pRoeUeU6j2/cAS", + "hne8NEWw+HB+JJ6mxyP8Ons2ev7Nt9MRn4t0hNnh0dPjr5/Rm+AOnv5bFevGHXt2JkxoN2DZWyy4l0sE", + "+gwrlPnCQ6YtmIILLMniFIV0UitHaqmqZLMrVugVS1iJqazKUfehfbuQOcUt/LnpGrRd2FfRIvd46yWF", + "el/TS1mi87w0sFqgavMEVtxBJq2j2OTSeaSkSFimbck9m7GUexyFMx/gJVRLabUqB9PgZfsRCj7HIniq", + "m7GtlcbqtBJ1avTELJAXfnHrPPeV6ws6rawlIXEZxGX71dHEgmxaMwqwytH5YHyl+JLLgs8L3HV+s/ij", + "jii487cL5NbPkfsPhUNnQIthuxgsCpTLPxCFWOz34AV97PliayCq5Si8Ha3Q+dHhkPcd2qUUeEsfBrz/", + "RjpPAuplEJaBX0hXZ5zgCozVS5liV/YVW5YxDp5LhZY8Lz2WQURPifoFt5avw7M2UtwOW/7TyeUFhAXR", + "+ph2jT479qeiHIe344d4ojLp76y4EPK4/ffGuXWCnv+KwpNCAYApAH11znkuVQDMog5Q0MYNQ2/4b+v9", + "f1rM2Iz9Y9I2q0kN+5OI+QMRUXjnbw3P8dbr31ANOIheh2BY9FbiUqo8uIl2Au0kLS26qghqPtT+twHH", + "bGgsb/FdhcPIHT7AXKfroEPTn9q9Pc/8DXrAIwHwF4T5FISBJ2XlPPUh62El/QJamDl4OPwY7j1aEvef", + "VJTX13Hl9fV4/NVgdbQU7yoGbDc9diza93lNeG4GauyltdoOlMGrU3j+zfQ5EEYUkisPSCupgI1WDnvl", + "lKLnsuif9LoquRoR+lEDBrwzBVehIsEZFDKTAryO/tVChGYvcCiPpHKe07d+qr49A4sZhq1tuLbnR9V3", + "Tt9idGXlaLt3MH3v4SWvLy/PGzYidNrZK5XHHG1IM+mLAYUvFtp6WOy6xlVlye26KTZj9bzAQVyILz7s", + "hZpBrxsYjj4IOx9u/V7e1bujUUPZ9LqhPZ+K1C256sO0clWJ9rbg+UCpVuU8zkqVMlYLdA5TKNE5nqMD", + "2enU/3IQCrs5sFuq045LpPLPju8JZs0BPsIAQ/K1DPBJRTqRraVW2mslhfRrEAsUvx3cRxc+HIgdp3QV", + "6wclALCorPTrC2ry0atz5BbtSRVH0fj0qtHjh18uWbJn4A+/XELo/KDnAW9TyKwug3uFVpnMK4sp0Ilw", + "HsGaPBx4BakTRbSGLbw3ceKUKtN1oD0XlDKbfeEvTn+EQAbg5PwMRtDlBMBVCiVXPI/dWWfQ7Yxhmxtf", + "q/jPdiyKyF1r760uqMErDKc5VCkYtFKnUrSBpFNYwgopkABw9r6Zw08MFwuEozHNnpUtavPcbDJZrVZj", + "Hj6Ptc0n9V43eXN2+vKni5ejo/F0vPBl0cEKtrWUJWyJ1kUfLA95YRb8kFZqg4obyWbs6Xg6Po7dZBEC", + "O2kJX46D9ecrqxzwLXFsB8WaQ0bX6LCDF5DJwmPIwiA4ev0srTv3yZZ2du46rnp3BuGM5vz5eneYa243", + "3lVo1+31xu5c2L3U+IRpr1dH+6r9yO9kWZWgtkhSc1TKgMBb71Gv5HeREDv5X9xRL8WMV4Vns8PpNKF1", + "JKB5kqp+6iNMX7mWVJtI+CODG1Knw827uuzbf0NAElt4SJKj6bQpvpplcmMKKYKoya9OB6rfnvfR4SHM", + "K6Gw91peJQids6qAbRZRLh9/UIG6CX71aYpEUjOgxHc8BVu3pvCxDtRjyf9Z4Z1BQXMb1msSVvf9hgrz", + "5notTEk8p3qKYxy72STM3DO21LDGQeGqbqra1kMpcAV4J50nMqAVjq/VDoRKR3ShNJpcQOVJOTW7ViM4", + "yyIFTjU6UNrHY5IdMai8XdMZ8coqhSdH08OD7u6wySWRhzSr63k5IG7bLCVBtENPh0wPAuDugs5pEHJS", + "TyJ1ML/T6frPTeOhsXOz24Zpjt/81eU0lEWxQTT+e2Jx1B10Dyizj6aHj6XHNuqUET1FHrW8z9SSFzIF", + "qUzlo/RvH096dEdIeJoVi9Cduu019FUOqcwC4fbQjmwJFWsYf6nj888RnFqIUduLgH142iQNAZm8r3+d", + "2NxLRb5Hvz2LQEd611z/n73o1f336Jui36MaQ8a2Sybdn13++vb32ba+pjbOXoCrSAymUYfjR68Q7SHT", + "lUo/xyzvJuUDE3yy88sAIU8v1evh477BArwGiybcCURiTE2xGfKg4Pn4Wm1na9iOew64xXaqnBGfLnD/", + "u8yVtpiOB5pp0Ot1d/b+g6X153fj3pXC59KF23hwIdD4pp7+v/3uSzV3q/mCCq5/w9Sr6s4lScj77vXI", + "1Q2ltgs/VMeqiCP+hBs5aafym+2xg8W/c0XR1KDb+y2fbW42/wsAAP//", +} + +// decodeSpec returns the embedded OpenAPI spec as raw JSON bytes, +// after base64-decoding and flate-decompressing the embedded blob. +func decodeSpec() ([]byte, error) { + encoded := strings.Join(swaggerSpec, "") + compressed, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return nil, fmt.Errorf("error base64 decoding spec: %w", err) + } + zr := flate.NewReader(bytes.NewReader(compressed)) + var buf bytes.Buffer + if _, err := buf.ReadFrom(zr); err != nil { + return nil, fmt.Errorf("read flate: %w", err) + } + if err := zr.Close(); err != nil { + return nil, fmt.Errorf("close flate reader: %w", err) + } + + return buf.Bytes(), nil +} + +var rawSpec = decodeSpecCached() + +// a naive cache of the decoded OpenAPI spec +func decodeSpecCached() func() ([]byte, error) { + data, err := decodeSpec() + return func() ([]byte, error) { + return data, err + } +} + +// Constructs a synthetic filesystem for resolving external references when loading openapi specifications. +func PathToRawSpec(pathToFile string) map[string]func() ([]byte, error) { + res := make(map[string]func() ([]byte, error)) + if len(pathToFile) > 0 { + res[pathToFile] = rawSpec + } + + return res +} + +// GetSpec returns the OpenAPI specification corresponding to the generated +// code in this file. External references in the spec are resolved through +// PathToRawSpec; externally-referenced files must be embedded in their +// corresponding Go packages (via the import-mapping feature). URL-based +// external refs are not supported. +func GetSpec() (swagger *openapi3.T, err error) { + resolvePath := PathToRawSpec("") + + loader := openapi3.NewLoader() + loader.IsExternalRefsAllowed = true + loader.ReadFromURIFunc = func(loader *openapi3.Loader, url *url.URL) ([]byte, error) { + pathToFile := url.String() + pathToFile = path.Clean(pathToFile) + getSpec, ok := resolvePath[pathToFile] + if !ok { + err1 := fmt.Errorf("path not found: %s", pathToFile) + return nil, err1 + } + return getSpec() + } + var specData []byte + specData, err = rawSpec() + if err != nil { + return + } + swagger, err = loader.LoadFromData(specData) + if err != nil { + return + } + return +} + +// GetSpecJSON returns the raw JSON bytes of the embedded OpenAPI +// specification: decompressed but not unmarshaled. External references +// are not resolved here; the bytes are the spec exactly as embedded by +// codegen. The result is cached at package init time, so repeated calls +// are cheap. +func GetSpecJSON() ([]byte, error) { + return rawSpec() +} + +// GetSwagger returns the OpenAPI specification corresponding to the +// generated code in this file. +// +// Deprecated: GetSwagger predates kin-openapi renaming openapi3.Swagger +// to openapi3.T. Use [GetSpec] instead. This wrapper is retained for +// backwards compatibility. +func GetSwagger() (*openapi3.T, error) { + return GetSpec() +} diff --git a/api/sp/v1alpha1/provider/types.gen.cfg b/api/agent/v1alpha1/types.gen.cfg similarity index 76% rename from api/sp/v1alpha1/provider/types.gen.cfg rename to api/agent/v1alpha1/types.gen.cfg index 52613dd..8d09a89 100644 --- a/api/sp/v1alpha1/provider/types.gen.cfg +++ b/api/agent/v1alpha1/types.gen.cfg @@ -1,4 +1,4 @@ -package: provider +package: agent generate: models: true output-options: diff --git a/api/agent/v1alpha1/types.gen.go b/api/agent/v1alpha1/types.gen.go new file mode 100644 index 0000000..bb07324 --- /dev/null +++ b/api/agent/v1alpha1/types.gen.go @@ -0,0 +1,230 @@ +// Package agent provides primitives to interact with the openapi HTTP API. +// +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.7.0 DO NOT EDIT. +package agent + +import ( + "time" +) + +const ( + BearerAuthScopes bearerAuthContextKey = "bearerAuth.Scopes" +) + +// Defines values for AgentCost. +const ( + AgentCostHigh AgentCost = "high" + AgentCostLow AgentCost = "low" + AgentCostMedium AgentCost = "medium" + AgentCostMediumHigh AgentCost = "medium-high" + AgentCostMediumLow AgentCost = "medium-low" +) + +// Valid indicates whether the value is a known member of the AgentCost enum. +func (e AgentCost) Valid() bool { + switch e { + case AgentCostHigh: + return true + case AgentCostLow: + return true + case AgentCostMedium: + return true + case AgentCostMediumHigh: + return true + case AgentCostMediumLow: + return true + default: + return false + } +} + +// Defines values for AgentHealthStatus. +const ( + AgentHealthStatusCongested AgentHealthStatus = "congested" + AgentHealthStatusReady AgentHealthStatus = "ready" + AgentHealthStatusUnavailable AgentHealthStatus = "unavailable" +) + +// Valid indicates whether the value is a known member of the AgentHealthStatus enum. +func (e AgentHealthStatus) Valid() bool { + switch e { + case AgentHealthStatusCongested: + return true + case AgentHealthStatusReady: + return true + case AgentHealthStatusUnavailable: + return true + default: + return false + } +} + +// Defines values for AgentRegistrationRequestCost. +const ( + AgentRegistrationRequestCostHigh AgentRegistrationRequestCost = "high" + AgentRegistrationRequestCostLow AgentRegistrationRequestCost = "low" + AgentRegistrationRequestCostMedium AgentRegistrationRequestCost = "medium" + AgentRegistrationRequestCostMediumHigh AgentRegistrationRequestCost = "medium-high" + AgentRegistrationRequestCostMediumLow AgentRegistrationRequestCost = "medium-low" +) + +// Valid indicates whether the value is a known member of the AgentRegistrationRequestCost enum. +func (e AgentRegistrationRequestCost) Valid() bool { + switch e { + case AgentRegistrationRequestCostHigh: + return true + case AgentRegistrationRequestCostLow: + return true + case AgentRegistrationRequestCostMedium: + return true + case AgentRegistrationRequestCostMediumHigh: + return true + case AgentRegistrationRequestCostMediumLow: + return true + default: + return false + } +} + +// Defines values for ListAgentsParamsHealthStatus. +const ( + ListAgentsParamsHealthStatusCongested ListAgentsParamsHealthStatus = "congested" + ListAgentsParamsHealthStatusReady ListAgentsParamsHealthStatus = "ready" + ListAgentsParamsHealthStatusUnavailable ListAgentsParamsHealthStatus = "unavailable" +) + +// Valid indicates whether the value is a known member of the ListAgentsParamsHealthStatus enum. +func (e ListAgentsParamsHealthStatus) Valid() bool { + switch e { + case ListAgentsParamsHealthStatusCongested: + return true + case ListAgentsParamsHealthStatusReady: + return true + case ListAgentsParamsHealthStatusUnavailable: + return true + default: + return false + } +} + +// Agent Full agent resource representation +type Agent struct { + // AgentId Server-generated unique identifier + AgentId *string `json:"agent_id,omitempty"` + + // Cost Relative cost weight for placement decisions + Cost *AgentCost `json:"cost,omitempty"` + + // CreateTime Timestamp when the agent was first registered + CreateTime *time.Time `json:"create_time,omitempty"` + + // Environment Environment label for the agent + Environment *string `json:"environment,omitempty"` + + // HealthStatus Current health status of the agent + HealthStatus *AgentHealthStatus `json:"health_status,omitempty"` + + // LastHeartbeat Timestamp of last heartbeat received + LastHeartbeat *time.Time `json:"last_heartbeat,omitempty"` + + // Name Unique name of the agent + Name *string `json:"name,omitempty"` + + // ServiceTypes List of service types this agent can provide + ServiceTypes *[]string `json:"service_types,omitempty"` + + // TopicName NATS topic name for this agent + TopicName *string `json:"topic_name,omitempty"` + + // UpdateTime Timestamp when the agent was last updated + UpdateTime *time.Time `json:"update_time,omitempty"` +} + +// AgentCost Relative cost weight for placement decisions +type AgentCost string + +// AgentHealthStatus Current health status of the agent +type AgentHealthStatus string + +// AgentList Paginated list of agents +type AgentList struct { + Agents *[]Agent `json:"agents,omitempty"` + + // NextPageToken Token for retrieving the next page of results + NextPageToken *string `json:"next_page_token,omitempty"` +} + +// AgentRegistrationRequest Request body for agent registration +type AgentRegistrationRequest struct { + // Cost Relative cost weight for placement decisions + Cost AgentRegistrationRequestCost `json:"cost"` + + // Environment Environment label for the agent + Environment string `json:"environment"` + + // Name Unique name of the agent + Name string `json:"name"` + + // ServiceTypes List of service types this agent can provide + ServiceTypes []string `json:"service_types"` + + // TopicName NATS topic name for this agent (must start with dcm.agent.) + TopicName string `json:"topic_name"` +} + +// AgentRegistrationRequestCost Relative cost weight for placement decisions +type AgentRegistrationRequestCost string + +// Error RFC 7807 compliant error response +type Error struct { + // Detail Human-readable explanation specific to this occurrence + Detail *string `json:"detail,omitempty"` + + // Instance URI reference for this specific error occurrence + Instance *string `json:"instance,omitempty"` + + // Status HTTP status code + Status *int `json:"status,omitempty"` + + // Title Short human-readable summary of the problem + Title string `json:"title"` + + // Type URI reference identifying the error type + Type string `json:"type"` +} + +// HeartbeatRequest Request body for agent heartbeat +type HeartbeatRequest struct { + // ConsumerLag Number of unprocessed messages in the agent's NATS consumer + ConsumerLag int64 `json:"consumer_lag"` + + // Timestamp Timestamp of this heartbeat (used for monotonicity check) + Timestamp time.Time `json:"timestamp"` +} + +// AgentIdPath defines model for AgentIdPath. +type AgentIdPath = string + +// bearerAuthContextKey is the context key for bearerAuth security scheme +type bearerAuthContextKey string + +// ListAgentsParams defines parameters for ListAgents. +type ListAgentsParams struct { + // HealthStatus Filter agents by health status + HealthStatus *ListAgentsParamsHealthStatus `form:"health_status,omitempty" json:"health_status,omitempty"` + + // MaxPageSize Maximum number of results per page + MaxPageSize *int `form:"max_page_size,omitempty" json:"max_page_size,omitempty"` + + // PageToken Token for pagination + PageToken *string `form:"page_token,omitempty" json:"page_token,omitempty"` +} + +// ListAgentsParamsHealthStatus defines parameters for ListAgents. +type ListAgentsParamsHealthStatus string + +// CreateAgentJSONRequestBody defines body for CreateAgent for application/json ContentType. +type CreateAgentJSONRequestBody = AgentRegistrationRequest + +// AgentHeartbeatJSONRequestBody defines body for AgentHeartbeat for application/json ContentType. +type AgentHeartbeatJSONRequestBody = HeartbeatRequest diff --git a/api/sp/v1alpha1/provider/openapi.yaml b/api/sp/v1alpha1/provider/openapi.yaml deleted file mode 100644 index 360ae70..0000000 --- a/api/sp/v1alpha1/provider/openapi.yaml +++ /dev/null @@ -1,470 +0,0 @@ -openapi: 3.0.4 -info: - contact: {} - description: | - DCM Service Provider API - Registration and management of Service Providers. - This API implements the Service Provider registration flow as defined in the - DCM enhancement proposal. - title: Service Provider API - version: v1alpha1 - license: - name: Apache 2.0 - url: https://www.apache.org/licenses/LICENSE-2.0.html - -servers: - - url: /api/v1alpha1 - -security: - - bearerAuth: [] - -tags: - - name: provider - description: Service Provider management operations - -paths: - /providers: - get: - tags: - - provider - summary: List all providers - operationId: listProviders - description: Returns a list of registered service providers with optional filtering - parameters: - - name: type - in: query - description: Filter providers by service type - schema: - type: string - - name: max_page_size - in: query - description: Maximum number of results per page - schema: - type: integer - minimum: 1 - maximum: 100 - default: 100 - - name: page_token - in: query - description: Token for pagination - schema: - type: string - responses: - '200': - description: Successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/ProviderList' - '400': - description: Bad request - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - '401': - $ref: '#/components/responses/Unauthorized' - '403': - $ref: '#/components/responses/Forbidden' - default: - description: Unexpected error - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - - post: - tags: - - provider - summary: Register a Service Provider - operationId: createProvider - description: | - Register a new service provider or update an existing one. - Registration is idempotent: - - If name does not exist, a new SP entry is created - - If name exists with same/no providerID, the entry is updated - - If name exists with different providerID, registration fails (conflict) - - If providerID exists with different name, registration fails (conflict) - parameters: - - name: id - in: query - description: Optional provider ID for idempotent registration - schema: - type: string - pattern: '^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$' - minLength: 1 - maxLength: 63 - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/Provider' - responses: - '200': - description: Provider updated (re-registration) - content: - application/json: - schema: - $ref: '#/components/schemas/Provider' - '201': - description: Provider created (new registration) - content: - application/json: - schema: - $ref: '#/components/schemas/Provider' - '400': - description: Invalid input - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - '409': - description: Conflict - name or providerID already in use - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - '401': - $ref: '#/components/responses/Unauthorized' - '403': - $ref: '#/components/responses/Forbidden' - '422': - description: Validation exception - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - default: - description: Unexpected error - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - - /providers/{providerId}: - get: - tags: - - provider - summary: Get a provider - operationId: getProvider - description: Get a service provider by its unique ID - parameters: - - $ref: '#/components/parameters/ProviderIdPath' - responses: - '200': - description: Successful operation - content: - application/json: - schema: - $ref: '#/components/schemas/Provider' - '400': - description: Invalid ID supplied - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - '401': - $ref: '#/components/responses/Unauthorized' - '403': - $ref: '#/components/responses/Forbidden' - '404': - description: Provider not found - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - default: - description: Unexpected error - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - - put: - tags: - - provider - summary: Update a Service Provider - operationId: applyProvider - description: | - Update an existing service provider. - The providerID in the path must match an existing provider. - parameters: - - $ref: '#/components/parameters/ProviderIdPath' - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/Provider' - responses: - '200': - description: Provider updated successfully - content: - application/json: - schema: - $ref: '#/components/schemas/Provider' - '400': - description: Invalid input - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - '401': - $ref: '#/components/responses/Unauthorized' - '403': - $ref: '#/components/responses/Forbidden' - '404': - description: Provider not found - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - '409': - description: Conflict - name already in use by another provider - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - default: - description: Unexpected error - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - - delete: - tags: - - provider - summary: Delete a service Provider - operationId: deleteProvider - description: Remove a service provider from the registry - parameters: - - $ref: '#/components/parameters/ProviderIdPath' - responses: - '204': - description: Provider deleted successfully - '400': - description: Invalid ID supplied - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - '401': - $ref: '#/components/responses/Unauthorized' - '403': - $ref: '#/components/responses/Forbidden' - '404': - description: Provider not found - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - default: - description: Unexpected error - content: - application/problem+json: - schema: - $ref: '#/components/schemas/Error' - -components: - parameters: - ProviderIdPath: - name: providerId - in: path - required: true - schema: - type: string - pattern: '^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$' - minLength: 1 - maxLength: 63 - description: Unique identifier of the provider - example: "123e4567-e89b-12d3-a456-426614174000" - securitySchemes: - bearerAuth: - type: http - scheme: bearer - bearerFormat: JWT - description: JWT token obtained from the configured Auth Provider - - responses: - Unauthorized: - description: Unauthorized - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - example: - type: UNAUTHENTICATED - status: 401 - title: Authentication required - detail: Valid authentication credentials are required to access this resource - Forbidden: - description: Forbidden - content: - application/json: - schema: - $ref: '#/components/schemas/Error' - example: - type: PERMISSION_DENIED - status: 403 - title: Access forbidden - detail: User does not have permission to access this resource - - schemas: - ProviderMetadata: - type: object - description: Additional metadata about the provider - additionalProperties: true - properties: - region_code: - type: string - description: Geographic region code where the provider operates - example: "us-east-1" - zone: - type: string - description: Availability zone or datacenter identifier - example: "datacenter-b" - status: - type: string - description: Current operational status of the provider - example: "healthy" - resources: - $ref: '#/components/schemas/ResourceCapacity' - - ResourceCapacity: - type: object - description: Resource capacity information - properties: - total_cpu: - type: integer - description: Total CPU cores available - example: 200 - total_memory: - type: string - description: Total memory available - example: "1TB" - total_storage: - type: string - description: Total storage available - example: "2TB" - total_node: - type: integer - description: Total number of nodes - example: 100 - - Provider: - type: object - description: Full provider resource representation - x-aep-resource: - type: serviceprovider.dcm.io/provider - singular: provider - plural: providers - patterns: - - providers/{provider_id} - required: - - name - - endpoint - - service_type - - schema_version - properties: - id: - type: string - pattern: '^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$' - minLength: 1 - maxLength: 63 - readOnly: true - description: Unique identifier for the Service Provider - example: "123e4567-e89b-12d3-a456-426614174000" - path: - type: string - readOnly: true - description: Resource path identifier - example: "providers/123e4567-e89b-12d3-a456-426614174000" - name: - type: string - description: Unique name of the Service Provider - example: "kubevirt-123" - display_name: - type: string - description: Human-readable display name for the provider - example: "KubeVirt Service Provider" - endpoint: - type: string - format: uri - description: Full endpoint URL where the provider API is accessible - example: "https://sp1.example.com/api/v1alpha1/vm" - service_type: - type: string - description: Type of service this provider offers - example: "vm" - schema_version: - type: string - description: Schema version of the service type the SP supports - pattern: "^v[0-9]+(alpha|beta)?[0-9]*$" - example: "v1alpha1" - operations: - type: array - items: - type: string - description: List of operations supported for this service type - example: ["create", "delete", "update"] - metadata: - $ref: '#/components/schemas/ProviderMetadata' - health_status: - type: string - description: Health status of the provider - example: "ready" - readOnly: true - create_time: - type: string - format: date-time - readOnly: true - description: Timestamp when the provider was first registered - update_time: - type: string - format: date-time - readOnly: true - description: Timestamp when the provider was last updated - - ProviderList: - type: object - description: Paginated list of providers - properties: - providers: - type: array - items: - $ref: '#/components/schemas/Provider' - next_page_token: - type: string - description: Token for retrieving the next page of results - example: "eyJpZCI6IjEyM2U0NTY3LWU4OWItMTJkMy1hNDU2LTQyNjYxNDE3NDAwMCJ9" - - Error: - type: object - description: RFC 7807 compliant error response - required: - - type - - title - properties: - type: - type: string - format: uri-reference - description: URI reference identifying the error type - example: "https://dcm.example.com/errors/invalid-input" - title: - type: string - description: Short human-readable summary of the problem - example: "Invalid Input" - status: - type: integer - description: HTTP status code - example: 400 - detail: - type: string - description: Human-readable explanation specific to this occurrence - example: "The 'name' field is required but was not provided" - instance: - type: string - format: uri-reference - description: URI reference for this specific error occurrence - example: "/errors/123e4567-e89b-12d3-a456-426614174000" diff --git a/api/sp/v1alpha1/provider/spec.gen.go b/api/sp/v1alpha1/provider/spec.gen.go deleted file mode 100644 index 1c3fb1a..0000000 --- a/api/sp/v1alpha1/provider/spec.gen.go +++ /dev/null @@ -1,158 +0,0 @@ -// Package provider provides primitives to interact with the openapi HTTP API. -// -// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.7.0 DO NOT EDIT. -package provider - -import ( - "bytes" - "compress/flate" - "encoding/base64" - "fmt" - "net/url" - "path" - "strings" - - "github.com/getkin/kin-openapi/openapi3" -) - -// Base64 encoded, compressed with deflate, json marshaled OpenAPI spec. -// Stored as a slice of fixed-width chunks rather than one concatenated -// const string: with thousands of chunks the chained `+` fold is several -// times slower for the Go compiler than parsing a slice literal. -var swaggerSpec = []string{ - "7Fpvc9s20v8qGDydafJU/606jd90XNtJlbMdnS0303N8HohciUhIgAFA2UxO3/1mAZIiRcpWMznFM3fv", - "RALYXSx2f/vDUl+oJ6NYChBG04MvNGaKRWBA2aexkgvugxr5Y2YCfOOD9hSPDZeCHtArwT8lQLgPwvAZ", - "B0XkjJgASJwtpC0K9yyKQ6AHtD/Yg+HP+y/a8MvLabs/8PfabPjzfns42N/vD/svhr1ej7YoR8kx6mtR", - "wSJcGRd20BZV8CnhCnx6YFQCLaq9ACKGxkXs/hTEHC3d32vRiIv8sd9CiQYUyv7nNWt/7rVf3jzLfrRv", - "vvRa+/1l/v75rz/QFjVpjLq1UVzM6XK5RNU6lkKDdc4rqabc90HggyeFAWHwJ4vjkHsMXdT9oKUdLpyA", - "HjSMh+g8DYr4EjQR0pCALYDEoCKuNZeCGEmY54HWxARcEwVaJsoD2qLaMJNoejDs7bWo4cb69tDNnRU2", - "FfaPTy7ORpeXo7fnt8cn56OTY7os++wHBTN6QP+vu4qDrhvV3ROlpHI7r577auvLFr0SLDGBVPwznslX", - "ueIPFnKfoBgMJLeAeApsXLFQE6aA5Oe+lWv6JddUxRbhU3jo6vzwavL7yflkdHQ4+Sb+qbhkWcizYeMW", - "1VLp4tURefFL7wVBPSFnwhDAmSSPOdrCNIhBGe7iL/feuqTfk4iJtgLms2kIBO7jkAm3dx2Dx2fcQxda", - "30nPS5QCYb23ytRJAORHzL0fyYxD6BPr5sz908SQO+aiNktMv54umMfaMJRch42LEVEwA6sYY9YZU1jn", - "Nr7Btq4d1d0t0WQmVcQMPaCJ4u1CaZO9efDU/DmZjIkbJJ70K9YMe71CEhcG5qBQVBZ665IuA6kMCarn", - "o5MoYiotIec0hKiy5ZFY2AQZiTgxTaa7Fw+7OQPplIu5VeScbFeWdQXGxPqg2/W9qJO97Xgyyr3OnSlt", - "npmyrXuXZdy+ppla56ebYracfgDP4I7yylPf1askDIv6UiQ/URAr0CCMjfRasngKmIFbw6MGR014BNqw", - "KCZ3AYhKAbORPuNKG6JgzrUBBx3Fvn1moG3F4g6Z/1aEaV6Zaufkcx2HLL11Ze2RxM0mE5ycJcmGwvq3", - "ZAp/cGXIJagF94CMV7NqNoDwY8kdPjd4Nh8mVxen6A4FVX8cjkeIBg6A+TRsjh4d9yvRw2LeXfRZGAes", - "311Ea4HTZGYALDTB7caktMN5Wj7EOtCf6TaHw/1tCE5+EA2u/stM59vylUc3GIFhPjOPlrV8S2f5/GVO", - "xDa4x0ZodgYP+uVjMoUFV6bdH+w1HTpmrM3fhhM/5dqgktUcopM4lsqAX6ohmfp1XLvOEIBiqQ7B/khi", - "TF6EH24gsio3ICtlSrEUn+NGHnyRoxAOl6Klsvk8OrcuXY8eqDuw2wUozR25Wqs3dpxk4/kJlV3kjmyc", - "e1JXDM4TllaCcXGNIffTMzv2rykY9vxX++r/f2isqk7bbXOJmqANcrayCc+wwBo5m+FFpGJT1KTEneRX", - "onvItCFOwtci+1p1s8lSQto1L9QOrlYAW/S+zSBuF9TW3szsEWhUsAqlL/nPW+4vUVAcJoqF5XBDfVzM", - "k5Cp0uuSG51t+UAHKz+X3WLislSOMQnrHh6zORfoPhJmSVpWXi3EAu7NbczmcGvkR2gI2gm+tgmtwCgO", - "i5yw4EqCK1GBAp2Ea/EK6Zv4H0ej/dGHk/RscNU7n/y5d/ruavj23cicTd58PEv7wfnx1eB08vf0/MOf", - "9+fHJ3vnx4d3Z0dvXjZF1WoTB19WELENctaRY/kAxzkr4TLzfY6OYOG45DYXdFU/HRYzSQ7shE1lYtZL", - "YdX/yGKkuLVEtub71yDnisUB94ibZwlvEw1wKAzVA0h0G5g27X6TN/NYftSJOZgesZh53KQPcfMje0Mw", - "q6LAwm04gSMXaZOZn6VocMzhgvGQTXnITUpwCpGKoMs9EAbUJshfzWhPGzlxLSZqm99ca7xsCuHCYVYT", - "7zXSsPDWi5OmRDMsJEfjK+JJBZowt8cqoxtsuOBYsRFEUqWbJLvRZrG0P/mt8SJj5YrG4HRSRRJNXZ8J", - "Z1Xir/+QrdpIxeYbxWbDG6wdNFlbPz5b7rxEcZPa0uuOYApMgTpMHHFwT6/yIvPm3YSuZ/abdxNiwZHI", - "qWFcIL9RMrLh7Ekx4/ME7+EosUyzbP6gOU7FymAk5a5LgZGSt2iYh0hea10cH53VOJwl/W1yYS9ALs8I", - "Ez6JmGBziGwCzmqrdOe9mGA9t1cG9CTO1I0sMbtcZbJnobwjTBMfZnb33Fbt9wJtAxEw4TmlGOpSs7Dz", - "HgM/5B4IbU84axsexswLgAw6yKYSFZYuKHd3dx1mhztSzbvZWt09HR2dnF+etAedXicwUVi6zdMmt9AW", - "LejXijA5KitYzOkB3ev0OkPHoQIbEN1KYZmDaUpykyihCSsq6ur2WZClQgy54yYgMs4AcMZDAzZGS4x6", - "5GccelwuzaVu73XtOmjFlLRM03V6bVu1nxJQ6apXW6E4DZx62VpXdMbueZREpdTOqjuJUT9mbbOqiN07", - "KqH556pOH2YsCU2GCZFTkD9xkT3V0aJu3IqOxI7lOJRtMqfEah7a/81aH3nQ623RNt2uI1nhaQ2NycvE", - "3ttnSbgqmRitwwdtyHpSP/01WzZ2R39jvu0mgjZOd3+TqMJP3WpHFRftPb6o0qYuQmJX27wScB+Dh7QY", - "sjktmnX78vssK/WyMCENm1cIPr1BHip1I0Q4QCCMCLirgQISFHepIUwQuOfaIJGWAjrvRQXJuUb6EsUS", - "fXLwXrTJaOau9MXHCbu8lWm6HBMQRqW40F2q/fIiOzeDJM0i6ApZGDU6brneY74+u3ZtWu/zme0omoqE", - "aq1gPNTkGdbFkHvmeSZqNX+DQNT1qKgagB7Z/ZaK7oMI+jZH5OJQRscWSlb+rliwAVe4T3f3levGXWVB", - "m9+kn35zYMq/o5U/4S13AIhNCVrU8SwKyTMF7fJ5PEfcGDh42p01WU6RZ5hsNXN2itT5hwfX7d8dVg97", - "L3e3yaMs30k7a2SqMnqw0DaQkYMmGqxxg8HujLNfRh08wb0HcV6wn1o1K1Wjpo8Q9bK2bJV48KqNNfKX", - "rtTZHm1D0YvkAgirl7vihpRlTFrD7mMrczN2N7llNaW79l+IBho3bOiM5ea5DflEFwQsTL9bOo+ObcM3", - "5Hl+7iiph7vbauF4pC8zmQj/KWaNC8lSOD+cM63my+JrME0ZMU0JN5ok7hvN6LiWEK/B/AezYTc1/Elc", - "aP6XV08sr1xGxI8kE3Ka+ifN+p1pPbNsXwvKJIFnn5WYCUiUaEMiZrygIqS0uJaIh3Ecpt82Ff9LGfyT", - "KK+7Z8vfHQC+K2GvUnSse0xIE5R6l08Ro3Kg2ZYvlz4uWFQof1a4vsGsR5zKMcN1uyv/u7HIkEmu/VNg", - "vbVdbu2v/pRR+0MuXd4s/x0AAP//", -} - -// decodeSpec returns the embedded OpenAPI spec as raw JSON bytes, -// after base64-decoding and flate-decompressing the embedded blob. -func decodeSpec() ([]byte, error) { - encoded := strings.Join(swaggerSpec, "") - compressed, err := base64.StdEncoding.DecodeString(encoded) - if err != nil { - return nil, fmt.Errorf("error base64 decoding spec: %w", err) - } - zr := flate.NewReader(bytes.NewReader(compressed)) - var buf bytes.Buffer - if _, err := buf.ReadFrom(zr); err != nil { - return nil, fmt.Errorf("read flate: %w", err) - } - if err := zr.Close(); err != nil { - return nil, fmt.Errorf("close flate reader: %w", err) - } - - return buf.Bytes(), nil -} - -var rawSpec = decodeSpecCached() - -// a naive cache of the decoded OpenAPI spec -func decodeSpecCached() func() ([]byte, error) { - data, err := decodeSpec() - return func() ([]byte, error) { - return data, err - } -} - -// Constructs a synthetic filesystem for resolving external references when loading openapi specifications. -func PathToRawSpec(pathToFile string) map[string]func() ([]byte, error) { - res := make(map[string]func() ([]byte, error)) - if len(pathToFile) > 0 { - res[pathToFile] = rawSpec - } - - return res -} - -// GetSpec returns the OpenAPI specification corresponding to the generated -// code in this file. External references in the spec are resolved through -// PathToRawSpec; externally-referenced files must be embedded in their -// corresponding Go packages (via the import-mapping feature). URL-based -// external refs are not supported. -func GetSpec() (swagger *openapi3.T, err error) { - resolvePath := PathToRawSpec("") - - loader := openapi3.NewLoader() - loader.IsExternalRefsAllowed = true - loader.ReadFromURIFunc = func(loader *openapi3.Loader, url *url.URL) ([]byte, error) { - pathToFile := url.String() - pathToFile = path.Clean(pathToFile) - getSpec, ok := resolvePath[pathToFile] - if !ok { - err1 := fmt.Errorf("path not found: %s", pathToFile) - return nil, err1 - } - return getSpec() - } - var specData []byte - specData, err = rawSpec() - if err != nil { - return - } - swagger, err = loader.LoadFromData(specData) - if err != nil { - return - } - return -} - -// GetSpecJSON returns the raw JSON bytes of the embedded OpenAPI -// specification: decompressed but not unmarshaled. External references -// are not resolved here; the bytes are the spec exactly as embedded by -// codegen. The result is cached at package init time, so repeated calls -// are cheap. -func GetSpecJSON() ([]byte, error) { - return rawSpec() -} - -// GetSwagger returns the OpenAPI specification corresponding to the -// generated code in this file. -// -// Deprecated: GetSwagger predates kin-openapi renaming openapi3.Swagger -// to openapi3.T. Use [GetSpec] instead. This wrapper is retained for -// backwards compatibility. -func GetSwagger() (*openapi3.T, error) { - return GetSpec() -} diff --git a/api/sp/v1alpha1/provider/types.gen.go b/api/sp/v1alpha1/provider/types.gen.go deleted file mode 100644 index 929df2f..0000000 --- a/api/sp/v1alpha1/provider/types.gen.go +++ /dev/null @@ -1,258 +0,0 @@ -// Package provider provides primitives to interact with the openapi HTTP API. -// -// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.7.0 DO NOT EDIT. -package provider - -import ( - "encoding/json" - "fmt" - "time" -) - -const ( - BearerAuthScopes bearerAuthContextKey = "bearerAuth.Scopes" -) - -// Error RFC 7807 compliant error response -type Error struct { - // Detail Human-readable explanation specific to this occurrence - Detail *string `json:"detail,omitempty"` - - // Instance URI reference for this specific error occurrence - Instance *string `json:"instance,omitempty"` - - // Status HTTP status code - Status *int `json:"status,omitempty"` - - // Title Short human-readable summary of the problem - Title string `json:"title"` - - // Type URI reference identifying the error type - Type string `json:"type"` -} - -// Provider Full provider resource representation -type Provider struct { - // CreateTime Timestamp when the provider was first registered - CreateTime *time.Time `json:"create_time,omitempty"` - - // DisplayName Human-readable display name for the provider - DisplayName *string `json:"display_name,omitempty"` - - // Endpoint Full endpoint URL where the provider API is accessible - Endpoint string `json:"endpoint"` - - // HealthStatus Health status of the provider - HealthStatus *string `json:"health_status,omitempty"` - - // Id Unique identifier for the Service Provider - Id *string `json:"id,omitempty"` - - // Metadata Additional metadata about the provider - Metadata *ProviderMetadata `json:"metadata,omitempty"` - - // Name Unique name of the Service Provider - Name string `json:"name"` - - // Operations List of operations supported for this service type - Operations *[]string `json:"operations,omitempty"` - - // Path Resource path identifier - Path *string `json:"path,omitempty"` - - // SchemaVersion Schema version of the service type the SP supports - SchemaVersion string `json:"schema_version"` - - // ServiceType Type of service this provider offers - ServiceType string `json:"service_type"` - - // UpdateTime Timestamp when the provider was last updated - UpdateTime *time.Time `json:"update_time,omitempty"` -} - -// ProviderList Paginated list of providers -type ProviderList struct { - // NextPageToken Token for retrieving the next page of results - NextPageToken *string `json:"next_page_token,omitempty"` - Providers *[]Provider `json:"providers,omitempty"` -} - -// ProviderMetadata Additional metadata about the provider -type ProviderMetadata struct { - // RegionCode Geographic region code where the provider operates - RegionCode *string `json:"region_code,omitempty"` - - // Resources Resource capacity information - Resources *ResourceCapacity `json:"resources,omitempty"` - - // Status Current operational status of the provider - Status *string `json:"status,omitempty"` - - // Zone Availability zone or datacenter identifier - Zone *string `json:"zone,omitempty"` - AdditionalProperties map[string]interface{} `json:"-"` -} - -// ResourceCapacity Resource capacity information -type ResourceCapacity struct { - // TotalCpu Total CPU cores available - TotalCpu *int `json:"total_cpu,omitempty"` - - // TotalMemory Total memory available - TotalMemory *string `json:"total_memory,omitempty"` - - // TotalNode Total number of nodes - TotalNode *int `json:"total_node,omitempty"` - - // TotalStorage Total storage available - TotalStorage *string `json:"total_storage,omitempty"` -} - -// ProviderIdPath defines model for ProviderIdPath. -type ProviderIdPath = string - -// Forbidden RFC 7807 compliant error response -type Forbidden = Error - -// Unauthorized RFC 7807 compliant error response -type Unauthorized = Error - -// bearerAuthContextKey is the context key for bearerAuth security scheme -type bearerAuthContextKey string - -// ListProvidersParams defines parameters for ListProviders. -type ListProvidersParams struct { - // Type Filter providers by service type - Type *string `form:"type,omitempty" json:"type,omitempty"` - - // MaxPageSize Maximum number of results per page - MaxPageSize *int `form:"max_page_size,omitempty" json:"max_page_size,omitempty"` - - // PageToken Token for pagination - PageToken *string `form:"page_token,omitempty" json:"page_token,omitempty"` -} - -// CreateProviderParams defines parameters for CreateProvider. -type CreateProviderParams struct { - // Id Optional provider ID for idempotent registration - Id *string `form:"id,omitempty" json:"id,omitempty"` -} - -// CreateProviderJSONRequestBody defines body for CreateProvider for application/json ContentType. -type CreateProviderJSONRequestBody = Provider - -// ApplyProviderJSONRequestBody defines body for ApplyProvider for application/json ContentType. -type ApplyProviderJSONRequestBody = Provider - -// Getter for additional properties for ProviderMetadata. Returns the specified -// element and whether it was found -func (a ProviderMetadata) Get(fieldName string) (value interface{}, found bool) { - if a.AdditionalProperties != nil { - value, found = a.AdditionalProperties[fieldName] - } - return -} - -// Setter for additional properties for ProviderMetadata -func (a *ProviderMetadata) Set(fieldName string, value interface{}) { - if a.AdditionalProperties == nil { - a.AdditionalProperties = make(map[string]interface{}) - } - a.AdditionalProperties[fieldName] = value -} - -// Override default JSON handling for ProviderMetadata to handle AdditionalProperties -func (a *ProviderMetadata) UnmarshalJSON(b []byte) error { - object := make(map[string]json.RawMessage) - err := json.Unmarshal(b, &object) - if err != nil { - return err - } - - if raw, found := object["region_code"]; found { - err = json.Unmarshal(raw, &a.RegionCode) - if err != nil { - return fmt.Errorf("error reading 'region_code': %w", err) - } - delete(object, "region_code") - } - - if raw, found := object["resources"]; found { - err = json.Unmarshal(raw, &a.Resources) - if err != nil { - return fmt.Errorf("error reading 'resources': %w", err) - } - delete(object, "resources") - } - - if raw, found := object["status"]; found { - err = json.Unmarshal(raw, &a.Status) - if err != nil { - return fmt.Errorf("error reading 'status': %w", err) - } - delete(object, "status") - } - - if raw, found := object["zone"]; found { - err = json.Unmarshal(raw, &a.Zone) - if err != nil { - return fmt.Errorf("error reading 'zone': %w", err) - } - delete(object, "zone") - } - - if len(object) != 0 { - a.AdditionalProperties = make(map[string]interface{}) - for fieldName, fieldBuf := range object { - var fieldVal interface{} - err := json.Unmarshal(fieldBuf, &fieldVal) - if err != nil { - return fmt.Errorf("error unmarshaling field %s: %w", fieldName, err) - } - a.AdditionalProperties[fieldName] = fieldVal - } - } - return nil -} - -// Override default JSON handling for ProviderMetadata to handle AdditionalProperties -func (a ProviderMetadata) MarshalJSON() ([]byte, error) { - var err error - object := make(map[string]json.RawMessage) - - if a.RegionCode != nil { - object["region_code"], err = json.Marshal(a.RegionCode) - if err != nil { - return nil, fmt.Errorf("error marshaling 'region_code': %w", err) - } - } - - if a.Resources != nil { - object["resources"], err = json.Marshal(a.Resources) - if err != nil { - return nil, fmt.Errorf("error marshaling 'resources': %w", err) - } - } - - if a.Status != nil { - object["status"], err = json.Marshal(a.Status) - if err != nil { - return nil, fmt.Errorf("error marshaling 'status': %w", err) - } - } - - if a.Zone != nil { - object["zone"], err = json.Marshal(a.Zone) - if err != nil { - return nil, fmt.Errorf("error marshaling 'zone': %w", err) - } - } - - for fieldName, field := range a.AdditionalProperties { - object[fieldName], err = json.Marshal(field) - if err != nil { - return nil, fmt.Errorf("error marshaling '%s': %w", fieldName, err) - } - } - return json.Marshal(object) -} diff --git a/api/sp/v1alpha1/resource_manager/openapi.yaml b/api/sp/v1alpha1/resource_manager/openapi.yaml index d79d04b..75b65c7 100644 --- a/api/sp/v1alpha1/resource_manager/openapi.yaml +++ b/api/sp/v1alpha1/resource_manager/openapi.yaml @@ -30,14 +30,14 @@ paths: operationId: listInstances description: Returns a list of service type instances with optional filtering parameters: - - name: provider + - name: service_type in: query - description: Filter service provider + description: Filter instances by service type schema: type: string - - name: service_type + - name: agent_name in: query - description: Filter instances by service type + description: Filter instances by the agent managing them schema: type: string - name: show_deleted @@ -145,6 +145,12 @@ paths: application/problem+json: schema: $ref: '#/components/schemas/Error' + '503': + description: Service unavailable - agent required by the requested placement but no agent store configured + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Error' default: description: Unexpected error content: @@ -237,6 +243,12 @@ paths: application/problem+json: schema: $ref: '#/components/schemas/Error' + '422': + description: Provisioning error - failed to publish the delete request to the agent + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Error' default: description: Unexpected error content: @@ -295,11 +307,10 @@ components: x-aep-resource: type: serviceprovider.dcm.io/service-type-instance singular: service-type-instance - plural: service-type-instance + plural: service-type-instances patterns: - service-type-instances/{instance_id} required: - - provider_name - spec properties: id: @@ -314,11 +325,15 @@ components: type: string readOnly: true description: Resource path identifier - example: "service-type-instance/123e4567-e89b-12d3-a456-426614174000" - provider_name: + example: "service-type-instances/123e4567-e89b-12d3-a456-426614174000" + agent_name: type: string - description: Name of the provider - example: "kubevirt-123" + readOnly: true + nullable: true + description: | + Name of the agent managing this instance. Absent or null when the + instance was created without agent routing. + example: "env-agent-west-1" status: type: string readOnly: true @@ -331,9 +346,7 @@ components: Deletion status for deferred deletions. Absent for active instances. SCHEDULED indicates the instance is queued for cleanup. FAILED indicates the cleanup has exceeded maximum retries. - PENDING_PROVIDER indicates the instance is waiting for its - provider to become healthy before cleanup is retried. - enum: [SCHEDULED, FAILED, PENDING_PROVIDER] + enum: [SCHEDULED, FAILED] spec: type: object description: | diff --git a/api/sp/v1alpha1/resource_manager/spec.gen.go b/api/sp/v1alpha1/resource_manager/spec.gen.go index c5bcb3c..8efdd25 100644 --- a/api/sp/v1alpha1/resource_manager/spec.gen.go +++ b/api/sp/v1alpha1/resource_manager/spec.gen.go @@ -20,49 +20,50 @@ import ( // const string: with thousands of chunks the chained `+` fold is several // times slower for the Go compiler than parsing a slice literal. var swaggerSpec = []string{ - "7Fptc9s28v8qO/x3pu389WirTqM3N66ltMzEis+Sm+lFPgcilxISEmAAULKa0Xe/WYCkRIly3F7jy9zc", - "O5MEsA/Y/e1vV/7kBTJJpUBhtNf/5KVMsQQNKvvkC22YCNAPr5hZ0JsQdaB4argUXt+7EfxjhsBDFIZH", - "HBXICMwCgecbvYaH9yxJY/T6XvfkFHs/nD1r4o/PZ83uSXjaZL0fzpq9k7Ozbq/7rNfpdLyGx+nklOQ1", - "PMES2slLPbyGp/BjxhWGXt+oDBueDhaYMFIuYfevUMxJ07PThpdwUTx2G3SiQUVn//Mta/7eaT6//S7/", - "o3n7qdM4626K99//7Ruv4Zl1SrK1UVzMvc1mQ6J1KoVG65wXUs14GKKgh0AKg8LQnyxNYx4wclH7vZb2", - "c+kE8qBhPCbnaVQQStQgpIEFWyKkqBKuNZcCjAQWBKg1mAXXoFDLTFmHasNMpr1+r3Pa8Aw31rfnbm1U", - "6lTqfzW8vvTHY//16G4wHPnDgbfZ9dk3CiOv7/1fexsHbfdVt4dKSeUsr9771vRNw7sRLDMLqfjvdCd/", - "yhW/spiHQMdQILkNECi0ccViDUwhFPf+KNd0d1xTPbYMn9JDN6Pzm8kvw9HEvzif/CX+qbhkU55nw8Zt", - "Okil6xcX8OzHzjMgOTFnwgDSSihizmt4qZIpKsNd/BXe2z/plyxhoqmQhWwWI+B9GjPhbNcpBjziAbnQ", - "+k4GQaYU7mfqZIHwLeXetxBxjEOwbs7dP8sMrJiL2lTJJQ93nVmkS6NM2hrYuPZBYYRWMMWsU6bUzhl+", - "RLe2/arbj0STSKqEGa/vZYo3S6F1+hbBc+DPyeQK3EcIZFjRptfplCdxYXCOio7KQ2//pPFCKgOL6v3o", - "LEmYWhfImSo5izGpmOyLpU0QX6SZqVPdvXjYzTlIr7mYW0HOyXbnrqyFManut9thkLTyt61AJoXXuVOl", - "yXNVHuvezS5uv/Vysc5Pt+VqOXuPgSGLxqiWPMDJOkX/aBy9yOIYtFtpLSnrTgkKoDBVqFEYmwEHSRQo", - "ZAbvDE9qzp/wBLVhSQqrBYpKYbMZEHGlDbgjwl1nhMxg055JZrPwtYjXRbk6uLwQYySBd8cCcJAvKIKQ", - "EiYkV1M2Frt1C85nZKf9zALDlzgVhbq6BeOLX4aDm1fDAXAREhiirlrENXzMMMPQnhDEyESWtqbixbl/", - "uCv/DAumAe8DxBBDSNg9T7IEFBrFUbem4mo4Gvijn++url//6g+G1w/IXjFuKDZJODd6KnJsUQRWMwxk", - "grBAFpvFGmYYSbVVwqITiQxbU7piFFlCUVaa7DU8Z4XX8PZVovD77B3x8DHkxyEZQh68QNEL/r9Bhf5a", - "QvNZK9NaknddpBJ93jG3YlCehU06s1lc62Mx+vOK5aFw5/jgvoYjluAOftqlFe0+ZDNccmWa3ZPTWuhP", - "MbB8JQw5ncniqx2UcErtQXl+xUXNcvU1knEsVxTFUpQa6SxNpTIYVqBqKnJOAN/9ejlOMWjAhRSGcYHK", - "PQ6YYTOm0T1JBRdxpo37+r0L9APYPAYhY4ccD7Hz6+H54LfHXEaWhn8eMmOmDbgT/ixi7pWSamzkd3lQ", - "UhrefZNh2izJou11bAJpOqU2fnX7U/HnHQ83dGoaZ4rFxwKexHMxz2KmHlhS2OM+F/q3qOJy2a7ftamv", - "ia+4NoeXcMXmXJCHIeba0K2XFh0UwO0XejCY6M+x3rrSXHIQjynF1vQs8N7cpWyOd0Z+cE3SXqjQa4uZ", - "DryXBTOhnUA7SXOFOouNroQqrl+m/7jwz/z3w/XlyU1nNPnt9NWbm97rN765nLz8cLnuLkaDm5NXk7+v", - "R+9/ux8Nhqejwfnq8uLl81pqspdHlEgYZIqb9ZiMds6ZIVOoqJvYPr0o4vflm4m3jxEv30zA2g5yZhM7", - "hEjJxNVPKSI+z6iC04lwtYUt62dSx4nYKkzUzHUbXESyaLVYQBFw0IIMLi5hfAUleF8yweaYEEE4v/Kh", - "CRdEWwi0mKDKXX6VUVm/KpSKyvmEaDpt53QTtFwfL3gQxXIFTBNXscZziwdTQaqhWNAaK5ECUmoWO0yL", - "eYDU7vQ/Fd3/ecqCBcJJi2pFpuIdlrparVrMfm5JNW/ne3X7lX8xHI2HzZNWp7UwSbxDykvoLlz+3fjq", - "+2N+8hreEpV2Ll12WZwuWJcOkykKlnKv7522Oq2e54qnjZP6DLaf5mjq6qvJlNDAymytpbQaVtwsQKau", - "QEHEY4M2gq02yl6mH3p9j0DB3034nZnO2wMObY8pRe6UTzuH+ZihWm8HMelemNoW+SCfjsjYWjJbV2w8", - "Iixfcpcv+QMC/Qhs6QAtI9O0FNnGXyGfKfJrEGdhGZYF0EwFi6WYax5izqJ3MgAGGDFaRZw0YrFGF7O1", - "2i/k6i4XXdE+dGd4fXtAmd0zKYnO1plzmfNqkSUzN2PLlYUUlUXKIzok7N5BsOa/Y70SXepgc+JePHGR", - "Px32tofKbWE8dWXHtVm10bOtBg9d5+3eoO2k03nEXOlxI5tjNbRmiDPO7JApymIo84tSv/egOnn//v9/", - "TK2jk6SfWGgnL6iNk909dlTpsnZ1+kSbTj+/qTLSK6Pjqcy8EXifYkBZivmahpdPRnJEA3as2SeMM2xu", - "eVxJmG6pbZB17MgWPgQGAldbciojYJXjW1PxOsfaeJ3z/DUwCGKOwjSZ1nxOVc0fTMWSM4sh73j4DmzQ", - "Qwm6LfCjyqys4aogyUIFKx7HMEdBAYbABPgDhylVTHdK7/SSD4J6oTj4A5uZtW47kqS8ilZfdqZ+67g8", - "avOTDNdfMsuLAf7ubwebA6DpfnkV9ipVEX/FFOmpAaaYLbqB3tNBTK/TezojR9JAJDOR6/r86SRfSBHF", - "PDDQLPPOHwCLqcddE/XItO2eeicnT6eU/bnFkX+8DzAtKtvXBvsFUosHYLoe+zeNYzR829L74cYVB6Jo", - "dbQ8kcuK8Ba8WaCAd8Xs9R1wnfPMYgoLEeNxphzFnAqFgVQ5x2TluNJOWW3b5XinxT9qgm0boB3taIFt", - "trieikxjCGFGkAkKF+tQ5dOmWK4sE2VLyUOYxTL44GZPJY13btV1FcVOlR+oKHUXul3S3vtp+JAXWl89", - "4J3tCJssqDrHDa6RhY7rkmPIMCacPS24ecglxzl5IfKP8fFDPto7DJcSyItuQ5f0MV7/x1DdH9jxY8wL", - "mP7vw/bS8WIX5L82KHPZtosmx0hr7YzgZzQVGJytgRsNmfsJwh8cZPfPaL5caj+mvXZZm7/OM7nX6bWm", - "4kl66CfuIb/a/vF/QPCVAcF+Ju+N+47QmZ2JtM3g3Vn021uKdtdOuvx2M9I2S3l7O7O8LU8++HGofkxc", - "Bq0+/Ecsb3O7+VcAAAD//w==", + "5Fpvc9s20v8qO3w603Ye/bVVp9GbG9dyWmZixxfLzfRinwORSwkpCDAAKFn16LvfLEBSlEQ5bq7xZe7e", + "mSKwu1js/n67S98HkUozJVFaEwzvg4xplqJF7Z5CaSyTEYbxBbMz+iVGE2meWa5kMAyuJP+YI/AYpeUJ", + "Rw0qATtD4MXGoBXgHUszgcEw6B8c4uCHo2dt/PH5pN0/iA/bbPDDUXtwcHTUH/SfDXq9XtAKOEnOSF8r", + "kCylnbyyI2gFGj/mXGMcDK3OsRWYaIYpI+NSdvcK5ZQsPTpsBSmX5WO/RRItapL9z3es/Uev/fzmu+KP", + "9s19r3XUX5W/f/+3b4JWYJcZ6TZWczkNVqsVqTaZkgadc14oPeFxjJIeIiUtSkt/siwTPGLkou4Ho9zr", + "ygnkQcu4IOcZ1BArNCCVhRmbI2SoU24MVxKsAhZFaAzYGTeg0ahcO4cay2xuguGgd9gKLLfOt8d+bVLZ", + "VNl/cfrmLLy8DF+f345Oz8PTUbCq++wbjUkwDP6vu46Drn9ruqdaK+1Pvnnv66OvWsGVZLmdKc3/oDv5", + "LFf8ygSPgcRQIPkNEGl0ccWEAaYRynt/lGv6Nddsiq3Cp/LQ1fnx1fiX0/NxeHI8/kv8s+GSVSXPhY3f", + "tJNKb16cwLMfe8+A9AjOpAWklVDGXNAKMq0y1Jb7+Cu9ty3plzxlsq2RxWwiEPAuE0z6s5sMI57wiFzo", + "fKeiKNcatzN1PEP4lnLvW0g4ihicmwv3T3ILC+ajNtNqzuO6M8t0aVVJ2wAbb0LQmKBTTDHrjams8wff", + "Y1vXvTXdR6JJonTKbDAMcs3bldIme8vg2fHneHwB/iVEKt6wZtDrVZK4tDhFTaKK0NuWdDlT2sJs835M", + "nqZML0vkzLSaCEw3jhzKuUuQUGa5bTLd//CwmwuQXnI5dYq8k93Ouq6ZtZkZdrtxlHaKXzuRSkuvc29K", + "mxemPNa9qzpuvwsKtd5PN9VqNfmAkaUTXaKe8wjHywzDvXH0IhcCjF/pTlLxTgUKoDHTaFBalwE7ScSm", + "KO2tp5lt8ecsxfJa3DpImWRT70BuKmUdOJ6QBlAaJJm0mKGkXdeysocSJtLILMaw4HamclvI1Cq3XE47", + "13LjHlDO225Be4HGtvtEhrkQFDIl8VEIvZZiWT7vhIVXeGt50+nGPEVjWZpV5sKGtQnXxpY21286Zhbb", + "TuYjTIhRICm83Zddo2JBmWGEBjHFEUFNudtULqbXLLJ8XnOu6cDlyS+no6tXpyPgMiakR7N5Im7gY445", + "xk5CJJDJPOtcyxfH4e6u4jXMmAG8ixBjjCFldzzNU9BoNUdT3JfMU4rnSn/QCrxIiupPeofHj6mpPEAi", + "FDkBlBQQ/hsV1l9bJ33ylFlj7fimzFB6XTvuxoGK5G6TzHZ134/F/k9aRozjUCCOOZnFxEUNHfymLQgv", + "7qDkKs+riRJCLQgYlKwgw+RZpjQlfB2irmVRC8B3v55dZhi14ERJy7hE7R9HzLIJM+iflIYTkRvr337v", + "w24HLvdl16VPqoeq8jenx6PfHuOsPIs/H00EMxa8hM8Fky0KcXe3Qx2t4K7NMGtXRaHraVxEG7erOaDu", + "yz9vebwiqZnINRN7I5BqTS6nuWB635oaB/rXRa2kO0StXHWbd62aye8VN3bX6xfERo5TBDeWrrlu4SbT", + "rd/Qg8XUfKq8beLgqtgImNZsSc8S7+xtxqZ4a9Xvvhvaig362aGYx855WYLQTqCdZLlGkwtrNklw+TL7", + "x0l4FH44XZ4dXPXOx78dvnp7NXj9NrRn45e/ny37s/PR1cGr8d+X5x9+uzsfnR6ej44XZycvnzfWIFuJ", + "Q5mDUa65XV7Sob1zJsg0amob1k8vyoB9+XYcbIPCy7djcGcHNXGZHEOiVeq5RMmET3NiM5IIF0UUBEVT", + "QOZ4FWuDqQbzbQWXiSp7KhZRBOz0GqOTM7i8gApOz6hGwZTI8vgihDacEIUTSjEZ+wrGv1VJxSgbtRNR", + "25jqG9rO6SZoudlPQZAItQBmiLfd4XlR/ZBpKGe0xmmkgFSGCQ9igkdIfc3wvmzzjzMWzRAOOgTeuRa1", + "cnSxWHSYe91Retot9pruq/Dk9PzytH3Q6XVmNhW16rvC6tLl311efL/PT0ErmKM23qXzPhPZjPVJmMpQ", + "sowHw+Cw0+sMAk9nLk66e7BheB9M0TYxns21NMCqbG2sXY0rEEFlnpEg4cKii2BnjXaXGcbBMCBQCOsJ", + "XxvevNsplp2YmpbJckN/OXn5mKNerkcvxZLbYsm6Md5JrscobCylXbfTpLtWnP8pzWECjkHAqMS2XRHp", + "orI0hGnydiTyuArWEn6uJRNKTg2Psagza3kBI0wYraL+OWHCoI/kRr/N1OK2UL1hfexlBEMnoMr5iVJU", + "dTYd56yoPGWeTvyIrTAWMtQOP/fYkLI7D8yG/4HNRvSpgS1K2/KJy+Jpt7XdNW4N7pknI99lNZlT44iH", + "rvNma8520Os9Yqz0uInNPmZtmOFc5m7GlOQCqqwjQBg8aE7Rvv//nzNr7yDpJxa7wQsa63X394mqXNbd", + "HD7RpsNPb9qY6FXR8VTHvJJ4l2FEWYrFmlZQDEYKnAO2r9cn5LNs6sq7qoy6ob5DNdVMjg4RGEhcrGtU", + "lQDbEN+5lq8LBBbLotxfAoNIcGrKmTF8SlwXjq7lnDOHIe95/B5c0EMFxR0Ik41RWctzI+lCDQsuBExR", + "UoAhMAnhyGPKJtJ7o2s934NQXxoO4chlZqPb9iQp30SrLztSv/ElPRr7k4qXXzLLy/l9/dPBagdo+l/e", + "hC2mKuOvnLM8NcCUo0U/z3s6iBn0Bk93yHNlIVG5LGx9/nSaT5RMBI8stKu8C0fABLW6Syo9cuN6qsHB", + "wdMZ5b62+JYA7yLMSmb7wd/i0xhRFue5ZHPG3VwT2uVItPrUsCxqM4cRGEMmWNFLTHILUhUbjFW63mp9", + "jSRW8o58gHSamWzV2tdqrOcWYbzyVEcFZ1Prkar5hvIOvJ2hhPflrPU9cFNUzeXUFRLGRa59wXwtNUZK", + "FxUzq2akbqrqWsvaTVGj71od44uoDriGkptrmRuMIc6JAEDjbBnrYoQm1MLV1WyueAwToaLf/UCtZE7t", + "3Wqa+NFNkR/gx6YLXS/pbn3n3q1yna8e8M56ZE0n2HSOH1Qji33lTo6hgzHpz9OBq4dcsr/DKFX+ue5i", + "t7oe7IZLRUtl72SqYlgs/2McFY7cTFXwknT++5iqcrzcoKynZAc3LTFcuSD1HwnbLtZ9cGf5RHAzc9nu", + "o6NKevdRuejvv0YE9iBRB8F9nUPj+OZntBvoPVkCtwZy/70mHO2A0s9ovxwiPWbG4cGm+LkAoEFv0LmW", + "TzLIeOJG/qtt4v/n8etrA4LtTN6axO6pwmofC1wG1z8TvLuhaPc9vc9vP77usox31+Pkm0ryzoe65gl+", + "FbRm95/hgtXN6l8BAAD//w==", } // decodeSpec returns the embedded OpenAPI spec as raw JSON bytes, diff --git a/api/sp/v1alpha1/resource_manager/types.gen.go b/api/sp/v1alpha1/resource_manager/types.gen.go index 6154ddb..52b9bd9 100644 --- a/api/sp/v1alpha1/resource_manager/types.gen.go +++ b/api/sp/v1alpha1/resource_manager/types.gen.go @@ -13,9 +13,8 @@ const ( // Defines values for ServiceTypeInstanceDeletionStatus. const ( - FAILED ServiceTypeInstanceDeletionStatus = "FAILED" - PENDINGPROVIDER ServiceTypeInstanceDeletionStatus = "PENDING_PROVIDER" - SCHEDULED ServiceTypeInstanceDeletionStatus = "SCHEDULED" + FAILED ServiceTypeInstanceDeletionStatus = "FAILED" + SCHEDULED ServiceTypeInstanceDeletionStatus = "SCHEDULED" ) // Valid indicates whether the value is a known member of the ServiceTypeInstanceDeletionStatus enum. @@ -23,8 +22,6 @@ func (e ServiceTypeInstanceDeletionStatus) Valid() bool { switch e { case FAILED: return true - case PENDINGPROVIDER: - return true case SCHEDULED: return true default: @@ -52,14 +49,16 @@ type Error struct { // ServiceTypeInstance Full service type instance resource representation type ServiceTypeInstance struct { + // AgentName Name of the agent managing this instance. Absent or null when the + // instance was created without agent routing. + AgentName *string `json:"agent_name,omitempty"` + // CreateTime Timestamp when the instance was first created CreateTime *time.Time `json:"create_time,omitempty"` // DeletionStatus Deletion status for deferred deletions. Absent for active // instances. SCHEDULED indicates the instance is queued for cleanup. // FAILED indicates the cleanup has exceeded maximum retries. - // PENDING_PROVIDER indicates the instance is waiting for its - // provider to become healthy before cleanup is retried. DeletionStatus *ServiceTypeInstanceDeletionStatus `json:"deletion_status,omitempty"` // Id Unique identifier for the Service Type Instance @@ -68,9 +67,6 @@ type ServiceTypeInstance struct { // Path Resource path identifier Path *string `json:"path,omitempty"` - // ProviderName Name of the provider - ProviderName string `json:"provider_name"` - // Spec Service specification following one of the supported service type // schemas (VMSpec, ContainerSpec, DatabaseSpec, or ClusterSpec). Spec map[string]interface{} `json:"spec"` @@ -85,8 +81,6 @@ type ServiceTypeInstance struct { // ServiceTypeInstanceDeletionStatus Deletion status for deferred deletions. Absent for active // instances. SCHEDULED indicates the instance is queued for cleanup. // FAILED indicates the cleanup has exceeded maximum retries. -// PENDING_PROVIDER indicates the instance is waiting for its -// provider to become healthy before cleanup is retried. type ServiceTypeInstanceDeletionStatus string // ServiceTypeInstanceList Paginated list of instances @@ -111,12 +105,12 @@ type bearerAuthContextKey string // ListInstancesParams defines parameters for ListInstances. type ListInstancesParams struct { - // Provider Filter service provider - Provider *string `form:"provider,omitempty" json:"provider,omitempty"` - // ServiceType Filter instances by service type ServiceType *string `form:"service_type,omitempty" json:"service_type,omitempty"` + // AgentName Filter instances by the agent managing them + AgentName *string `form:"agent_name,omitempty" json:"agent_name,omitempty"` + // ShowDeleted If true, soft-deleted instances are included in the results // alongside active instances. Defaults to false. ShowDeleted *bool `form:"show_deleted,omitempty" json:"show_deleted,omitempty"` diff --git a/go.mod b/go.mod index 71a4daf..d93dacf 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/coreos/go-oidc/v3 v3.20.0 github.com/getkin/kin-openapi v0.139.0 github.com/go-chi/chi/v5 v5.3.0 - github.com/go-resty/resty/v2 v2.17.2 github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.8.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/nats-io/nats-server/v2 v2.12.5 github.com/nats-io/nats.go v1.50.0 @@ -48,7 +48,6 @@ require ( github.com/gorilla/mux v1.8.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.8.0 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect diff --git a/go.sum b/go.sum index 4ff84fb..acc1789 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,6 @@ github.com/go-openapi/swag/jsonname v0.25.4 h1:bZH0+MsS03MbnwBXYhuTttMOqk+5KcQ98 github.com/go-openapi/swag/jsonname v0.25.4/go.mod h1:GPVEk9CWVhNvWhZgrnvRA6utbAltopbKwDu8mXNUMag= github.com/go-openapi/testify/v2 v2.0.2 h1:X999g3jeLcoY8qctY/c/Z8iBHTbwLz7R2WXd6Ub6wls= github.com/go-openapi/testify/v2 v2.0.2/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= -github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk= -github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= diff --git a/internal/sp/api/provider/server.gen.cfg b/internal/agent/api/server/server.gen.cfg similarity index 84% rename from internal/sp/api/provider/server.gen.cfg rename to internal/agent/api/server/server.gen.cfg index a7a4d34..af16a6d 100644 --- a/internal/sp/api/provider/server.gen.cfg +++ b/internal/agent/api/server/server.gen.cfg @@ -1,4 +1,4 @@ -package: provider +package: server generate: chi-server: true strict-server: true diff --git a/internal/agent/api/server/server.gen.go b/internal/agent/api/server/server.gen.go new file mode 100644 index 0000000..ea11a48 --- /dev/null +++ b/internal/agent/api/server/server.gen.go @@ -0,0 +1,1000 @@ +// Package server provides primitives to interact with the openapi HTTP API. +// +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.7.0 DO NOT EDIT. +package server + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + "github.com/go-chi/chi/v5" + "github.com/oapi-codegen/runtime" +) + +const ( + BearerAuthScopes bearerAuthContextKey = "bearerAuth.Scopes" +) + +// Defines values for AgentCost. +const ( + AgentCostHigh AgentCost = "high" + AgentCostLow AgentCost = "low" + AgentCostMedium AgentCost = "medium" + AgentCostMediumHigh AgentCost = "medium-high" + AgentCostMediumLow AgentCost = "medium-low" +) + +// Valid indicates whether the value is a known member of the AgentCost enum. +func (e AgentCost) Valid() bool { + switch e { + case AgentCostHigh: + return true + case AgentCostLow: + return true + case AgentCostMedium: + return true + case AgentCostMediumHigh: + return true + case AgentCostMediumLow: + return true + default: + return false + } +} + +// Defines values for AgentHealthStatus. +const ( + AgentHealthStatusCongested AgentHealthStatus = "congested" + AgentHealthStatusReady AgentHealthStatus = "ready" + AgentHealthStatusUnavailable AgentHealthStatus = "unavailable" +) + +// Valid indicates whether the value is a known member of the AgentHealthStatus enum. +func (e AgentHealthStatus) Valid() bool { + switch e { + case AgentHealthStatusCongested: + return true + case AgentHealthStatusReady: + return true + case AgentHealthStatusUnavailable: + return true + default: + return false + } +} + +// Defines values for AgentRegistrationRequestCost. +const ( + AgentRegistrationRequestCostHigh AgentRegistrationRequestCost = "high" + AgentRegistrationRequestCostLow AgentRegistrationRequestCost = "low" + AgentRegistrationRequestCostMedium AgentRegistrationRequestCost = "medium" + AgentRegistrationRequestCostMediumHigh AgentRegistrationRequestCost = "medium-high" + AgentRegistrationRequestCostMediumLow AgentRegistrationRequestCost = "medium-low" +) + +// Valid indicates whether the value is a known member of the AgentRegistrationRequestCost enum. +func (e AgentRegistrationRequestCost) Valid() bool { + switch e { + case AgentRegistrationRequestCostHigh: + return true + case AgentRegistrationRequestCostLow: + return true + case AgentRegistrationRequestCostMedium: + return true + case AgentRegistrationRequestCostMediumHigh: + return true + case AgentRegistrationRequestCostMediumLow: + return true + default: + return false + } +} + +// Defines values for ListAgentsParamsHealthStatus. +const ( + ListAgentsParamsHealthStatusCongested ListAgentsParamsHealthStatus = "congested" + ListAgentsParamsHealthStatusReady ListAgentsParamsHealthStatus = "ready" + ListAgentsParamsHealthStatusUnavailable ListAgentsParamsHealthStatus = "unavailable" +) + +// Valid indicates whether the value is a known member of the ListAgentsParamsHealthStatus enum. +func (e ListAgentsParamsHealthStatus) Valid() bool { + switch e { + case ListAgentsParamsHealthStatusCongested: + return true + case ListAgentsParamsHealthStatusReady: + return true + case ListAgentsParamsHealthStatusUnavailable: + return true + default: + return false + } +} + +// Agent Full agent resource representation +type Agent struct { + // AgentId Server-generated unique identifier + AgentId *string `json:"agent_id,omitempty"` + + // Cost Relative cost weight for placement decisions + Cost *AgentCost `json:"cost,omitempty"` + + // CreateTime Timestamp when the agent was first registered + CreateTime *time.Time `json:"create_time,omitempty"` + + // Environment Environment label for the agent + Environment *string `json:"environment,omitempty"` + + // HealthStatus Current health status of the agent + HealthStatus *AgentHealthStatus `json:"health_status,omitempty"` + + // LastHeartbeat Timestamp of last heartbeat received + LastHeartbeat *time.Time `json:"last_heartbeat,omitempty"` + + // Name Unique name of the agent + Name *string `json:"name,omitempty"` + + // ServiceTypes List of service types this agent can provide + ServiceTypes *[]string `json:"service_types,omitempty"` + + // TopicName NATS topic name for this agent + TopicName *string `json:"topic_name,omitempty"` + + // UpdateTime Timestamp when the agent was last updated + UpdateTime *time.Time `json:"update_time,omitempty"` +} + +// AgentCost Relative cost weight for placement decisions +type AgentCost string + +// AgentHealthStatus Current health status of the agent +type AgentHealthStatus string + +// AgentList Paginated list of agents +type AgentList struct { + Agents *[]Agent `json:"agents,omitempty"` + + // NextPageToken Token for retrieving the next page of results + NextPageToken *string `json:"next_page_token,omitempty"` +} + +// AgentRegistrationRequest Request body for agent registration +type AgentRegistrationRequest struct { + // Cost Relative cost weight for placement decisions + Cost AgentRegistrationRequestCost `json:"cost"` + + // Environment Environment label for the agent + Environment string `json:"environment"` + + // Name Unique name of the agent + Name string `json:"name"` + + // ServiceTypes List of service types this agent can provide + ServiceTypes []string `json:"service_types"` + + // TopicName NATS topic name for this agent (must start with dcm.agent.) + TopicName string `json:"topic_name"` +} + +// AgentRegistrationRequestCost Relative cost weight for placement decisions +type AgentRegistrationRequestCost string + +// Error RFC 7807 compliant error response +type Error struct { + // Detail Human-readable explanation specific to this occurrence + Detail *string `json:"detail,omitempty"` + + // Instance URI reference for this specific error occurrence + Instance *string `json:"instance,omitempty"` + + // Status HTTP status code + Status *int `json:"status,omitempty"` + + // Title Short human-readable summary of the problem + Title string `json:"title"` + + // Type URI reference identifying the error type + Type string `json:"type"` +} + +// HeartbeatRequest Request body for agent heartbeat +type HeartbeatRequest struct { + // ConsumerLag Number of unprocessed messages in the agent's NATS consumer + ConsumerLag int64 `json:"consumer_lag"` + + // Timestamp Timestamp of this heartbeat (used for monotonicity check) + Timestamp time.Time `json:"timestamp"` +} + +// AgentIdPath defines model for AgentIdPath. +type AgentIdPath = string + +// bearerAuthContextKey is the context key for bearerAuth security scheme +type bearerAuthContextKey string + +// ListAgentsParams defines parameters for ListAgents. +type ListAgentsParams struct { + // HealthStatus Filter agents by health status + HealthStatus *ListAgentsParamsHealthStatus `form:"health_status,omitempty" json:"health_status,omitempty"` + + // MaxPageSize Maximum number of results per page + MaxPageSize *int `form:"max_page_size,omitempty" json:"max_page_size,omitempty"` + + // PageToken Token for pagination + PageToken *string `form:"page_token,omitempty" json:"page_token,omitempty"` +} + +// ListAgentsParamsHealthStatus defines parameters for ListAgents. +type ListAgentsParamsHealthStatus string + +// CreateAgentJSONRequestBody defines body for CreateAgent for application/json ContentType. +type CreateAgentJSONRequestBody = AgentRegistrationRequest + +// AgentHeartbeatJSONRequestBody defines body for AgentHeartbeat for application/json ContentType. +type AgentHeartbeatJSONRequestBody = HeartbeatRequest + +// ServerInterface represents all server handlers. +type ServerInterface interface { + // List all agents + // (GET /agents) + ListAgents(w http.ResponseWriter, r *http.Request, params ListAgentsParams) + // Register an agent + // (POST /agents) + CreateAgent(w http.ResponseWriter, r *http.Request) + // Get an agent + // (GET /agents/{agentId}) + GetAgent(w http.ResponseWriter, r *http.Request, agentId AgentIdPath) + // Send agent heartbeat + // (PUT /agents/{agentId}/heartbeat) + AgentHeartbeat(w http.ResponseWriter, r *http.Request, agentId AgentIdPath) +} + +// Unimplemented server implementation that returns http.StatusNotImplemented for each endpoint. + +type Unimplemented struct{} + +// List all agents +// (GET /agents) +func (_ Unimplemented) ListAgents(w http.ResponseWriter, r *http.Request, params ListAgentsParams) { + w.WriteHeader(http.StatusNotImplemented) +} + +// Register an agent +// (POST /agents) +func (_ Unimplemented) CreateAgent(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) +} + +// Get an agent +// (GET /agents/{agentId}) +func (_ Unimplemented) GetAgent(w http.ResponseWriter, r *http.Request, agentId AgentIdPath) { + w.WriteHeader(http.StatusNotImplemented) +} + +// Send agent heartbeat +// (PUT /agents/{agentId}/heartbeat) +func (_ Unimplemented) AgentHeartbeat(w http.ResponseWriter, r *http.Request, agentId AgentIdPath) { + w.WriteHeader(http.StatusNotImplemented) +} + +// ServerInterfaceWrapper converts contexts to parameters. +type ServerInterfaceWrapper struct { + Handler ServerInterface + HandlerMiddlewares []MiddlewareFunc + ErrorHandlerFunc func(w http.ResponseWriter, r *http.Request, err error) +} + +type MiddlewareFunc func(http.Handler) http.Handler + +// ListAgents operation middleware +func (siw *ServerInterfaceWrapper) ListAgents(w http.ResponseWriter, r *http.Request) { + + var err error + _ = err + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + // Parameter object where we will unmarshal all parameters from the context + var params ListAgentsParams + + // ------------- Optional query parameter "health_status" ------------- + + err = runtime.BindQueryParameterWithOptions("form", true, false, "health_status", r.URL.Query(), ¶ms.HealthStatus, runtime.BindQueryParameterOptions{Type: "string", Format: ""}) + if err != nil { + var requiredError *runtime.RequiredParameterError + if errors.As(err, &requiredError) { + siw.ErrorHandlerFunc(w, r, &RequiredParamError{ParamName: "health_status"}) + } else { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "health_status", Err: err}) + } + return + } + + // ------------- Optional query parameter "max_page_size" ------------- + + err = runtime.BindQueryParameterWithOptions("form", true, false, "max_page_size", r.URL.Query(), ¶ms.MaxPageSize, runtime.BindQueryParameterOptions{Type: "integer", Format: ""}) + if err != nil { + var requiredError *runtime.RequiredParameterError + if errors.As(err, &requiredError) { + siw.ErrorHandlerFunc(w, r, &RequiredParamError{ParamName: "max_page_size"}) + } else { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "max_page_size", Err: err}) + } + return + } + + // ------------- Optional query parameter "page_token" ------------- + + err = runtime.BindQueryParameterWithOptions("form", true, false, "page_token", r.URL.Query(), ¶ms.PageToken, runtime.BindQueryParameterOptions{Type: "string", Format: ""}) + if err != nil { + var requiredError *runtime.RequiredParameterError + if errors.As(err, &requiredError) { + siw.ErrorHandlerFunc(w, r, &RequiredParamError{ParamName: "page_token"}) + } else { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "page_token", Err: err}) + } + return + } + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.ListAgents(w, r, params) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + +// CreateAgent operation middleware +func (siw *ServerInterfaceWrapper) CreateAgent(w http.ResponseWriter, r *http.Request) { + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.CreateAgent(w, r) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + +// GetAgent operation middleware +func (siw *ServerInterfaceWrapper) GetAgent(w http.ResponseWriter, r *http.Request) { + + var err error + _ = err + + // ------------- Path parameter "agentId" ------------- + var agentId AgentIdPath + + err = runtime.BindStyledParameterWithOptions("simple", "agentId", chi.URLParam(r, "agentId"), &agentId, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true, Type: "string", Format: ""}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "agentId", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.GetAgent(w, r, agentId) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + +// AgentHeartbeat operation middleware +func (siw *ServerInterfaceWrapper) AgentHeartbeat(w http.ResponseWriter, r *http.Request) { + + var err error + _ = err + + // ------------- Path parameter "agentId" ------------- + var agentId AgentIdPath + + err = runtime.BindStyledParameterWithOptions("simple", "agentId", chi.URLParam(r, "agentId"), &agentId, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true, Type: "string", Format: ""}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "agentId", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.AgentHeartbeat(w, r, agentId) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + +type UnescapedCookieParamError struct { + ParamName string + Err error +} + +func (e *UnescapedCookieParamError) Error() string { + return fmt.Sprintf("error unescaping cookie parameter '%s'", e.ParamName) +} + +func (e *UnescapedCookieParamError) Unwrap() error { + return e.Err +} + +type UnmarshalingParamError struct { + ParamName string + Err error +} + +func (e *UnmarshalingParamError) Error() string { + return fmt.Sprintf("Error unmarshaling parameter %s as JSON: %s", e.ParamName, e.Err.Error()) +} + +func (e *UnmarshalingParamError) Unwrap() error { + return e.Err +} + +type RequiredParamError struct { + ParamName string +} + +func (e *RequiredParamError) Error() string { + return fmt.Sprintf("Query argument %s is required, but not found", e.ParamName) +} + +type RequiredHeaderError struct { + ParamName string + Err error +} + +func (e *RequiredHeaderError) Error() string { + return fmt.Sprintf("Header parameter %s is required, but not found", e.ParamName) +} + +func (e *RequiredHeaderError) Unwrap() error { + return e.Err +} + +type InvalidParamFormatError struct { + ParamName string + Err error +} + +func (e *InvalidParamFormatError) Error() string { + return fmt.Sprintf("Invalid format for parameter %s: %s", e.ParamName, e.Err.Error()) +} + +func (e *InvalidParamFormatError) Unwrap() error { + return e.Err +} + +type TooManyValuesForParamError struct { + ParamName string + Count int +} + +func (e *TooManyValuesForParamError) Error() string { + return fmt.Sprintf("Expected one value for %s, got %d", e.ParamName, e.Count) +} + +// Handler creates http.Handler with routing matching OpenAPI spec. +func Handler(si ServerInterface) http.Handler { + return HandlerWithOptions(si, ChiServerOptions{}) +} + +type ChiServerOptions struct { + BaseURL string + BaseRouter chi.Router + Middlewares []MiddlewareFunc + ErrorHandlerFunc func(w http.ResponseWriter, r *http.Request, err error) +} + +// HandlerFromMux creates http.Handler with routing matching OpenAPI spec based on the provided mux. +func HandlerFromMux(si ServerInterface, r chi.Router) http.Handler { + return HandlerWithOptions(si, ChiServerOptions{ + BaseRouter: r, + }) +} + +func HandlerFromMuxWithBaseURL(si ServerInterface, r chi.Router, baseURL string) http.Handler { + return HandlerWithOptions(si, ChiServerOptions{ + BaseURL: baseURL, + BaseRouter: r, + }) +} + +// HandlerWithOptions creates http.Handler with additional options +func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handler { + r := options.BaseRouter + + if r == nil { + r = chi.NewRouter() + } + if options.ErrorHandlerFunc == nil { + options.ErrorHandlerFunc = func(w http.ResponseWriter, r *http.Request, err error) { + http.Error(w, err.Error(), http.StatusBadRequest) + } + } + wrapper := ServerInterfaceWrapper{ + Handler: si, + HandlerMiddlewares: options.Middlewares, + ErrorHandlerFunc: options.ErrorHandlerFunc, + } + + r.Group(func(r chi.Router) { + r.Get(options.BaseURL+"/agents", wrapper.ListAgents) + }) + r.Group(func(r chi.Router) { + r.Post(options.BaseURL+"/agents", wrapper.CreateAgent) + }) + r.Group(func(r chi.Router) { + r.Get(options.BaseURL+"/agents/{agentId}", wrapper.GetAgent) + }) + r.Group(func(r chi.Router) { + r.Put(options.BaseURL+"/agents/{agentId}/heartbeat", wrapper.AgentHeartbeat) + }) + + return r +} + +type ListAgentsRequestObject struct { + Params ListAgentsParams +} + +type ListAgentsResponseObject interface { + VisitListAgentsResponse(w http.ResponseWriter) error +} + +type ListAgents200JSONResponse AgentList + +func (response ListAgents200JSONResponse) VisitListAgentsResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, err := buf.WriteTo(w) + return err +} + +type ListAgents400ApplicationProblemPlusJSONResponse Error + +func (response ListAgents400ApplicationProblemPlusJSONResponse) VisitListAgentsResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(400) + _, err := buf.WriteTo(w) + return err +} + +type ListAgentsdefaultApplicationProblemPlusJSONResponse struct { + Body Error + StatusCode int +} + +func (response ListAgentsdefaultApplicationProblemPlusJSONResponse) VisitListAgentsResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response.Body); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(response.StatusCode) + _, err := buf.WriteTo(w) + return err +} + +type CreateAgentRequestObject struct { + Body *CreateAgentJSONRequestBody +} + +type CreateAgentResponseObject interface { + VisitCreateAgentResponse(w http.ResponseWriter) error +} + +type CreateAgent200JSONResponse Agent + +func (response CreateAgent200JSONResponse) VisitCreateAgentResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, err := buf.WriteTo(w) + return err +} + +type CreateAgent201JSONResponse Agent + +func (response CreateAgent201JSONResponse) VisitCreateAgentResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(201) + _, err := buf.WriteTo(w) + return err +} + +type CreateAgent400ApplicationProblemPlusJSONResponse Error + +func (response CreateAgent400ApplicationProblemPlusJSONResponse) VisitCreateAgentResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(400) + _, err := buf.WriteTo(w) + return err +} + +type CreateAgent409ApplicationProblemPlusJSONResponse Error + +func (response CreateAgent409ApplicationProblemPlusJSONResponse) VisitCreateAgentResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(409) + _, err := buf.WriteTo(w) + return err +} + +type CreateAgentdefaultApplicationProblemPlusJSONResponse struct { + Body Error + StatusCode int +} + +func (response CreateAgentdefaultApplicationProblemPlusJSONResponse) VisitCreateAgentResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response.Body); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(response.StatusCode) + _, err := buf.WriteTo(w) + return err +} + +type GetAgentRequestObject struct { + AgentId AgentIdPath `json:"agentId"` +} + +type GetAgentResponseObject interface { + VisitGetAgentResponse(w http.ResponseWriter) error +} + +type GetAgent200JSONResponse Agent + +func (response GetAgent200JSONResponse) VisitGetAgentResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, err := buf.WriteTo(w) + return err +} + +type GetAgent400ApplicationProblemPlusJSONResponse Error + +func (response GetAgent400ApplicationProblemPlusJSONResponse) VisitGetAgentResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(400) + _, err := buf.WriteTo(w) + return err +} + +type GetAgent404ApplicationProblemPlusJSONResponse Error + +func (response GetAgent404ApplicationProblemPlusJSONResponse) VisitGetAgentResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(404) + _, err := buf.WriteTo(w) + return err +} + +type GetAgentdefaultApplicationProblemPlusJSONResponse struct { + Body Error + StatusCode int +} + +func (response GetAgentdefaultApplicationProblemPlusJSONResponse) VisitGetAgentResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response.Body); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(response.StatusCode) + _, err := buf.WriteTo(w) + return err +} + +type AgentHeartbeatRequestObject struct { + AgentId AgentIdPath `json:"agentId"` + Body *AgentHeartbeatJSONRequestBody +} + +type AgentHeartbeatResponseObject interface { + VisitAgentHeartbeatResponse(w http.ResponseWriter) error +} + +type AgentHeartbeat200JSONResponse Agent + +func (response AgentHeartbeat200JSONResponse) VisitAgentHeartbeatResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, err := buf.WriteTo(w) + return err +} + +type AgentHeartbeat400ApplicationProblemPlusJSONResponse Error + +func (response AgentHeartbeat400ApplicationProblemPlusJSONResponse) VisitAgentHeartbeatResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(400) + _, err := buf.WriteTo(w) + return err +} + +type AgentHeartbeat404ApplicationProblemPlusJSONResponse Error + +func (response AgentHeartbeat404ApplicationProblemPlusJSONResponse) VisitAgentHeartbeatResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(404) + _, err := buf.WriteTo(w) + return err +} + +type AgentHeartbeatdefaultApplicationProblemPlusJSONResponse struct { + Body Error + StatusCode int +} + +func (response AgentHeartbeatdefaultApplicationProblemPlusJSONResponse) VisitAgentHeartbeatResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response.Body); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(response.StatusCode) + _, err := buf.WriteTo(w) + return err +} + +// StrictServerInterface represents all server handlers. +type StrictServerInterface interface { + // List all agents + // (GET /agents) + ListAgents(ctx context.Context, request ListAgentsRequestObject) (ListAgentsResponseObject, error) + // Register an agent + // (POST /agents) + CreateAgent(ctx context.Context, request CreateAgentRequestObject) (CreateAgentResponseObject, error) + // Get an agent + // (GET /agents/{agentId}) + GetAgent(ctx context.Context, request GetAgentRequestObject) (GetAgentResponseObject, error) + // Send agent heartbeat + // (PUT /agents/{agentId}/heartbeat) + AgentHeartbeat(ctx context.Context, request AgentHeartbeatRequestObject) (AgentHeartbeatResponseObject, error) +} + +type StrictHandlerFunc func(ctx context.Context, w http.ResponseWriter, r *http.Request, request any) (any, error) +type StrictMiddlewareFunc func(f StrictHandlerFunc, operationID string) StrictHandlerFunc + +type StrictHTTPServerOptions struct { + RequestErrorHandlerFunc func(w http.ResponseWriter, r *http.Request, err error) + ResponseErrorHandlerFunc func(w http.ResponseWriter, r *http.Request, err error) +} + +func NewStrictHandler(ssi StrictServerInterface, middlewares []StrictMiddlewareFunc) ServerInterface { + return &strictHandler{ssi: ssi, middlewares: middlewares, options: StrictHTTPServerOptions{ + RequestErrorHandlerFunc: func(w http.ResponseWriter, r *http.Request, err error) { + http.Error(w, err.Error(), http.StatusBadRequest) + }, + ResponseErrorHandlerFunc: func(w http.ResponseWriter, r *http.Request, err error) { + http.Error(w, err.Error(), http.StatusInternalServerError) + }, + }} +} + +func NewStrictHandlerWithOptions(ssi StrictServerInterface, middlewares []StrictMiddlewareFunc, options StrictHTTPServerOptions) ServerInterface { + return &strictHandler{ssi: ssi, middlewares: middlewares, options: options} +} + +type strictHandler struct { + ssi StrictServerInterface + middlewares []StrictMiddlewareFunc + options StrictHTTPServerOptions +} + +// ListAgents operation middleware +func (sh *strictHandler) ListAgents(w http.ResponseWriter, r *http.Request, params ListAgentsParams) { + var request ListAgentsRequestObject + + request.Params = params + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.ListAgents(ctx, request.(ListAgentsRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "ListAgents") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(ListAgentsResponseObject); ok { + if err := validResponse.VisitListAgentsResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + +// CreateAgent operation middleware +func (sh *strictHandler) CreateAgent(w http.ResponseWriter, r *http.Request) { + var request CreateAgentRequestObject + + var body CreateAgentJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.CreateAgent(ctx, request.(CreateAgentRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "CreateAgent") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(CreateAgentResponseObject); ok { + if err := validResponse.VisitCreateAgentResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + +// GetAgent operation middleware +func (sh *strictHandler) GetAgent(w http.ResponseWriter, r *http.Request, agentId AgentIdPath) { + var request GetAgentRequestObject + + request.AgentId = agentId + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.GetAgent(ctx, request.(GetAgentRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "GetAgent") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(GetAgentResponseObject); ok { + if err := validResponse.VisitGetAgentResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + +// AgentHeartbeat operation middleware +func (sh *strictHandler) AgentHeartbeat(w http.ResponseWriter, r *http.Request, agentId AgentIdPath) { + var request AgentHeartbeatRequestObject + + request.AgentId = agentId + + var body AgentHeartbeatJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.AgentHeartbeat(ctx, request.(AgentHeartbeatRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "AgentHeartbeat") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(AgentHeartbeatResponseObject); ok { + if err := validResponse.VisitAgentHeartbeatResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} diff --git a/internal/agent/handlers/v1alpha1/convert.go b/internal/agent/handlers/v1alpha1/convert.go new file mode 100644 index 0000000..6e454c2 --- /dev/null +++ b/internal/agent/handlers/v1alpha1/convert.go @@ -0,0 +1,40 @@ +package v1alpha1 + +import ( + api "github.com/dcm-project/control-plane/api/agent/v1alpha1" + server "github.com/dcm-project/control-plane/internal/agent/api/server" +) + +func apiAgentToServer(a *api.Agent) server.Agent { + s := server.Agent{ + AgentId: a.AgentId, + Name: a.Name, + Environment: a.Environment, + TopicName: a.TopicName, + CreateTime: a.CreateTime, + UpdateTime: a.UpdateTime, + LastHeartbeat: a.LastHeartbeat, + } + if a.Cost != nil { + cost := server.AgentCost(*a.Cost) + s.Cost = &cost + } + if a.HealthStatus != nil { + hs := server.AgentHealthStatus(*a.HealthStatus) + s.HealthStatus = &hs + } + if a.ServiceTypes != nil { + st := make([]string, len(*a.ServiceTypes)) + copy(st, *a.ServiceTypes) + s.ServiceTypes = &st + } + return s +} + +func apiAgentsToServer(agents []api.Agent) []server.Agent { + out := make([]server.Agent, len(agents)) + for i := range agents { + out[i] = apiAgentToServer(&agents[i]) + } + return out +} diff --git a/internal/agent/handlers/v1alpha1/errors.go b/internal/agent/handlers/v1alpha1/errors.go new file mode 100644 index 0000000..46b9402 --- /dev/null +++ b/internal/agent/handlers/v1alpha1/errors.go @@ -0,0 +1,94 @@ +package v1alpha1 + +import ( + "context" + "errors" + "log/slog" + + server "github.com/dcm-project/control-plane/internal/agent/api/server" + "github.com/dcm-project/control-plane/internal/agent/service" +) + +// logServiceError logs at Warn for a *service.ServiceError that maps to a +// 4xx response (service.IsClientError) and Error otherwise, so severity +// matches whether the internalErrorDetail response below is hiding a real +// failure. A ServiceError code with no 4xx mapping (e.g. +// ErrCodeNotImplemented, which none of these mappers currently handle and +// which falls through to a 500 below) is deliberately logged at Error, not +// Warn, since it also represents a hidden failure from the caller's view. +func logServiceError(ctx context.Context, msg string, err error, attrs ...any) { + args := append([]any{"error", err}, attrs...) + var svcErr *service.ServiceError + if service.IsClientError(err, &svcErr) { + slog.WarnContext(ctx, msg, args...) + return + } + slog.ErrorContext(ctx, msg, args...) +} + +// newError builds an RFC 7807 error body. errType is a short slug for the +// "type" field, distinct from svcErr.Code (the longer internal URI). +func newError(errType, title, detail string, status int) server.Error { + return server.Error{Type: errType, Title: title, Detail: &detail, Status: &status} +} + +// internalErrorDetail is the fixed 5xx response detail: the real error is +// logged server-side by logServiceError before these functions are called, +// so the client-facing body never echoes it back. +const internalErrorDetail = "an internal error occurred" + +// createErrorResponse uses errors.As, not a raw type assertion, so a +// *service.ServiceError wrapped by fmt.Errorf("...: %w", err) is still recognized. +func createErrorResponse(err error) (server.CreateAgentResponseObject, error) { + var svcErr *service.ServiceError + if errors.As(err, &svcErr) { + switch svcErr.Code { + case service.ErrCodeValidation: + return server.CreateAgent400ApplicationProblemPlusJSONResponse( + newError("validation-error", "Invalid request", svcErr.Message, 400)), nil + case service.ErrCodeConflict: + return server.CreateAgent409ApplicationProblemPlusJSONResponse( + newError("conflict", "Agent already registered", svcErr.Message, 409)), nil + } + } + return server.CreateAgentdefaultApplicationProblemPlusJSONResponse{ + Body: newError("create-error", "Failed to register agent", internalErrorDetail, 500), + StatusCode: 500, + }, nil +} + +func getErrorResponse(err error) (server.GetAgentResponseObject, error) { + var svcErr *service.ServiceError + if errors.As(err, &svcErr) && svcErr.Code == service.ErrCodeNotFound { + return server.GetAgent404ApplicationProblemPlusJSONResponse( + newError("not-found", "Agent not found", svcErr.Message, 404)), nil + } + return server.GetAgentdefaultApplicationProblemPlusJSONResponse{ + Body: newError("get-error", "Failed to get agent", internalErrorDetail, 500), + StatusCode: 500, + }, nil +} + +func hbErrorResponse(err error) (server.AgentHeartbeatResponseObject, error) { + var svcErr *service.ServiceError + if errors.As(err, &svcErr) && svcErr.Code == service.ErrCodeNotFound { + return server.AgentHeartbeat404ApplicationProblemPlusJSONResponse( + newError("not-found", "Agent not found", svcErr.Message, 404)), nil + } + return server.AgentHeartbeatdefaultApplicationProblemPlusJSONResponse{ + Body: newError("heartbeat-error", "Failed to record heartbeat", internalErrorDetail, 500), + StatusCode: 500, + }, nil +} + +func listErrorResponse(err error) (server.ListAgentsResponseObject, error) { + var svcErr *service.ServiceError + if errors.As(err, &svcErr) && svcErr.Code == service.ErrCodeValidation { + return server.ListAgents400ApplicationProblemPlusJSONResponse( + newError("validation-error", "Invalid request", svcErr.Message, 400)), nil + } + return server.ListAgentsdefaultApplicationProblemPlusJSONResponse{ + Body: newError("list-error", "Failed to list agents", internalErrorDetail, 500), + StatusCode: 500, + }, nil +} diff --git a/internal/agent/handlers/v1alpha1/errors_test.go b/internal/agent/handlers/v1alpha1/errors_test.go new file mode 100644 index 0000000..052e32b --- /dev/null +++ b/internal/agent/handlers/v1alpha1/errors_test.go @@ -0,0 +1,112 @@ +package v1alpha1 + +import ( + "bytes" + "context" + "log/slog" + "strings" + + server "github.com/dcm-project/control-plane/internal/agent/api/server" + "github.com/dcm-project/control-plane/internal/agent/service" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("error response mappers", func() { + Describe("createErrorResponse", func() { + It("maps a validation error to 400", func() { + resp, err := createErrorResponse(service.NewValidationError("bad name")) + Expect(err).NotTo(HaveOccurred()) + typed, ok := resp.(server.CreateAgent400ApplicationProblemPlusJSONResponse) + Expect(ok).To(BeTrue()) + Expect(typed.Type).To(Equal("validation-error")) + }) + + It("maps a conflict error to a typed 409, not the generic default", func() { + resp, err := createErrorResponse(service.NewConflictError("already registered")) + Expect(err).NotTo(HaveOccurred()) + typed, ok := resp.(server.CreateAgent409ApplicationProblemPlusJSONResponse) + Expect(ok).To(BeTrue()) + Expect(typed.Type).To(Equal("conflict")) + }) + + It("still maps unrecognized errors to the generic 500 default", func() { + resp, err := createErrorResponse(service.NewNotImplementedError()) + Expect(err).NotTo(HaveOccurred()) + def, ok := resp.(server.CreateAgentdefaultApplicationProblemPlusJSONResponse) + Expect(ok).To(BeTrue()) + Expect(def.StatusCode).To(Equal(500)) + Expect(def.Body.Detail).To(HaveValue(Equal(internalErrorDetail))) + }) + }) + + Describe("getErrorResponse", func() { + It("maps a not-found error to 404", func() { + resp, err := getErrorResponse(service.NewNotFoundError("agent not found")) + Expect(err).NotTo(HaveOccurred()) + _, ok := resp.(server.GetAgent404ApplicationProblemPlusJSONResponse) + Expect(ok).To(BeTrue()) + }) + + It("maps an unrecognized error to the generic 500 default", func() { + resp, err := getErrorResponse(service.NewValidationError("shouldn't happen here")) + Expect(err).NotTo(HaveOccurred()) + def, ok := resp.(server.GetAgentdefaultApplicationProblemPlusJSONResponse) + Expect(ok).To(BeTrue()) + Expect(def.StatusCode).To(Equal(500)) + }) + }) + + Describe("hbErrorResponse", func() { + It("maps a not-found error to 404", func() { + resp, err := hbErrorResponse(service.NewNotFoundError("agent not found")) + Expect(err).NotTo(HaveOccurred()) + _, ok := resp.(server.AgentHeartbeat404ApplicationProblemPlusJSONResponse) + Expect(ok).To(BeTrue()) + }) + }) + + Describe("listErrorResponse", func() { + It("maps a validation error to 400", func() { + resp, err := listErrorResponse(service.NewValidationError("bad page token")) + Expect(err).NotTo(HaveOccurred()) + _, ok := resp.(server.ListAgents400ApplicationProblemPlusJSONResponse) + Expect(ok).To(BeTrue()) + }) + }) + + Describe("logServiceError", func() { + // Captures level via a text handler rather than asserting on + // ServiceError type directly, so this exercises the same + // service.IsClientError path the mappers above rely on. + logLevel := func(err error) string { + var buf bytes.Buffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + defer slog.SetDefault(prev) + + logServiceError(context.Background(), "op failed", err) + out := buf.String() + switch { + case strings.Contains(out, "level=WARN"): + return "WARN" + case strings.Contains(out, "level=ERROR"): + return "ERROR" + default: + return "UNKNOWN" + } + } + + It("logs a mapped 4xx ServiceError at Warn", func() { + Expect(logLevel(service.NewConflictError("already registered"))).To(Equal("WARN")) + }) + + It("logs a ServiceError with no 4xx mapping (e.g. not-implemented) at Error, since it's a hidden failure too", func() { + Expect(logLevel(service.NewNotImplementedError())).To(Equal("ERROR")) + }) + + It("logs a raw, non-ServiceError failure at Error", func() { + Expect(logLevel(context.DeadlineExceeded)).To(Equal("ERROR")) + }) + }) +}) diff --git a/internal/agent/handlers/v1alpha1/handler.go b/internal/agent/handlers/v1alpha1/handler.go new file mode 100644 index 0000000..a1a9cec --- /dev/null +++ b/internal/agent/handlers/v1alpha1/handler.go @@ -0,0 +1,93 @@ +// Package v1alpha1 implements HTTP handlers for the Agent API. +package v1alpha1 + +import ( + "context" + + api "github.com/dcm-project/control-plane/api/agent/v1alpha1" + server "github.com/dcm-project/control-plane/internal/agent/api/server" + "github.com/dcm-project/control-plane/internal/agent/service" +) + +type Handler struct { + agentService *service.AgentService +} + +func NewHandler(agentService *service.AgentService) *Handler { + return &Handler{agentService: agentService} +} + +var _ server.StrictServerInterface = (*Handler)(nil) + +func (h *Handler) ListAgents(ctx context.Context, req server.ListAgentsRequestObject) (server.ListAgentsResponseObject, error) { + var healthStatus string + if req.Params.HealthStatus != nil { + healthStatus = string(*req.Params.HealthStatus) + } + pageSize := 0 + if req.Params.MaxPageSize != nil { + pageSize = *req.Params.MaxPageSize + } + var pageToken string + if req.Params.PageToken != nil { + pageToken = *req.Params.PageToken + } + + result, err := h.agentService.List(ctx, healthStatus, pageSize, pageToken) + if err != nil { + logServiceError(ctx, "ListAgents failed", err) + return listErrorResponse(err) + } + + sAgents := apiAgentsToServer(result.Agents) + resp := server.AgentList{Agents: &sAgents} + if result.NextPageToken != "" { + resp.NextPageToken = &result.NextPageToken + } + return server.ListAgents200JSONResponse(resp), nil +} + +func (h *Handler) CreateAgent(ctx context.Context, req server.CreateAgentRequestObject) (server.CreateAgentResponseObject, error) { + apiReq := api.AgentRegistrationRequest{ + Name: req.Body.Name, + TopicName: req.Body.TopicName, + ServiceTypes: req.Body.ServiceTypes, + Environment: req.Body.Environment, + Cost: api.AgentRegistrationRequestCost(req.Body.Cost), + } + + agent, created, err := h.agentService.RegisterOrUpdate(ctx, apiReq) + if err != nil { + logServiceError(ctx, "RegisterOrUpdate failed", err) + return createErrorResponse(err) + } + + sa := apiAgentToServer(agent) + if created { + return server.CreateAgent201JSONResponse(sa), nil + } + return server.CreateAgent200JSONResponse(sa), nil +} + +func (h *Handler) GetAgent(ctx context.Context, req server.GetAgentRequestObject) (server.GetAgentResponseObject, error) { + agent, err := h.agentService.Get(ctx, req.AgentId) + if err != nil { + logServiceError(ctx, "GetAgent failed", err, "agent_id", req.AgentId) + return getErrorResponse(err) + } + return server.GetAgent200JSONResponse(apiAgentToServer(agent)), nil +} + +func (h *Handler) AgentHeartbeat(ctx context.Context, req server.AgentHeartbeatRequestObject) (server.AgentHeartbeatResponseObject, error) { + apiReq := api.HeartbeatRequest{ + ConsumerLag: req.Body.ConsumerLag, + Timestamp: req.Body.Timestamp, + } + + agent, err := h.agentService.Heartbeat(ctx, req.AgentId, apiReq) + if err != nil { + logServiceError(ctx, "AgentHeartbeat failed", err, "agent_id", req.AgentId) + return hbErrorResponse(err) + } + return server.AgentHeartbeat200JSONResponse(apiAgentToServer(agent)), nil +} diff --git a/internal/sp/service/provider/provider_suite_test.go b/internal/agent/handlers/v1alpha1/handler_suite_test.go similarity index 52% rename from internal/sp/service/provider/provider_suite_test.go rename to internal/agent/handlers/v1alpha1/handler_suite_test.go index 3d5c55b..c8d1c1d 100644 --- a/internal/sp/service/provider/provider_suite_test.go +++ b/internal/agent/handlers/v1alpha1/handler_suite_test.go @@ -1,4 +1,4 @@ -package provider_test +package v1alpha1_test import ( "testing" @@ -7,7 +7,7 @@ import ( . "github.com/onsi/gomega" ) -func TestProviderService(t *testing.T) { +func TestAgentHandler(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Provider Service Suite") + RunSpecs(t, "Agent Handler Suite") } diff --git a/internal/agent/handlers/v1alpha1/handler_test.go b/internal/agent/handlers/v1alpha1/handler_test.go new file mode 100644 index 0000000..c5b7a2e --- /dev/null +++ b/internal/agent/handlers/v1alpha1/handler_test.go @@ -0,0 +1,214 @@ +package v1alpha1_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + + api "github.com/dcm-project/control-plane/api/agent/v1alpha1" + server "github.com/dcm-project/control-plane/internal/agent/api/server" + handler "github.com/dcm-project/control-plane/internal/agent/handlers/v1alpha1" + "github.com/dcm-project/control-plane/internal/agent/service" + "github.com/dcm-project/control-plane/internal/agent/store/model" + "github.com/go-chi/chi/v5" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + agentstore "github.com/dcm-project/control-plane/internal/agent/store/agent" +) + +var _ = Describe("Agent Handler", func() { + var ( + db *gorm.DB + router *chi.Mux + ) + + BeforeEach(func() { + var err error + db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(db.AutoMigrate(&model.Agent{})).To(Succeed()) + + store := agentstore.NewAgent(db) + svc := service.NewAgentService(store, 100) + h := handler.NewHandler(svc) + + strictHandler := server.NewStrictHandler(h, nil) + router = chi.NewRouter() + server.HandlerFromMux(strictHandler, router) + }) + + AfterEach(func() { + sqlDB, _ := db.DB() + _ = sqlDB.Close() + }) + + Describe("POST /agents", func() { + It("returns 201 on create", func() { + body := `{"name":"test-agent","topic_name":"dcm.agent.test-agent","service_types":["vm"]}` + req := httptest.NewRequest(http.MethodPost, "/agents", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + + router.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusCreated)) + + var agent api.Agent + Expect(json.NewDecoder(rec.Body).Decode(&agent)).To(Succeed()) + Expect(agent.AgentId).NotTo(BeNil()) + }) + + It("returns 200 on re-registration", func() { + body := `{"name":"rereg-agent","topic_name":"dcm.agent.rereg-agent","service_types":["vm"]}` + + req1 := httptest.NewRequest(http.MethodPost, "/agents", strings.NewReader(body)) + req1.Header.Set("Content-Type", "application/json") + rec1 := httptest.NewRecorder() + router.ServeHTTP(rec1, req1) + Expect(rec1.Code).To(Equal(http.StatusCreated)) + + req2 := httptest.NewRequest(http.MethodPost, "/agents", strings.NewReader(body)) + req2.Header.Set("Content-Type", "application/json") + rec2 := httptest.NewRecorder() + router.ServeHTTP(rec2, req2) + + Expect(rec2.Code).To(Equal(http.StatusOK)) + }) + + It("returns 400 on invalid body", func() { + req := httptest.NewRequest(http.MethodPost, "/agents", strings.NewReader(`{invalid`)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + + router.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusBadRequest)) + }) + + It("returns 409 with an RFC 7807 body when another agent already owns the topic_name (K)", func() { + body1 := `{"name":"topic-owner","topic_name":"dcm.agent.shared-topic","service_types":["vm"]}` + req1 := httptest.NewRequest(http.MethodPost, "/agents", strings.NewReader(body1)) + req1.Header.Set("Content-Type", "application/json") + rec1 := httptest.NewRecorder() + router.ServeHTTP(rec1, req1) + Expect(rec1.Code).To(Equal(http.StatusCreated)) + + body2 := `{"name":"topic-squatter","topic_name":"dcm.agent.shared-topic","service_types":["vm"]}` + req2 := httptest.NewRequest(http.MethodPost, "/agents", strings.NewReader(body2)) + req2.Header.Set("Content-Type", "application/json") + rec2 := httptest.NewRecorder() + router.ServeHTTP(rec2, req2) + + Expect(rec2.Code).To(Equal(http.StatusConflict)) + Expect(rec2.Header().Get("Content-Type")).To(Equal("application/problem+json")) + + var problem server.Error + Expect(json.NewDecoder(rec2.Body).Decode(&problem)).To(Succeed()) + Expect(problem.Status).NotTo(BeNil()) + Expect(*problem.Status).To(Equal(409)) + }) + }) + + Describe("GET /agents/{agentId}", func() { + It("returns 200 on found", func() { + body := `{"name":"get-agent","topic_name":"dcm.agent.get-agent","service_types":["vm"]}` + reqCreate := httptest.NewRequest(http.MethodPost, "/agents", strings.NewReader(body)) + reqCreate.Header.Set("Content-Type", "application/json") + recCreate := httptest.NewRecorder() + router.ServeHTTP(recCreate, reqCreate) + Expect(recCreate.Code).To(Equal(http.StatusCreated)) + + var created api.Agent + Expect(json.NewDecoder(recCreate.Body).Decode(&created)).To(Succeed()) + + reqGet := httptest.NewRequest(http.MethodGet, "/agents/"+*created.AgentId, nil) + recGet := httptest.NewRecorder() + router.ServeHTTP(recGet, reqGet) + + Expect(recGet.Code).To(Equal(http.StatusOK)) + }) + + It("returns 404 on not found", func() { + req := httptest.NewRequest(http.MethodGet, "/agents/"+uuid.New().String(), nil) + rec := httptest.NewRecorder() + + router.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusNotFound)) + }) + }) + + Describe("GET /agents", func() { + It("returns 200 with list", func() { + req := httptest.NewRequest(http.MethodGet, "/agents", nil) + rec := httptest.NewRecorder() + + router.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusOK)) + }) + + It("returns an RFC 7807 problem body on internal failure instead of a raw error", func() { + // Force the store call inside List to fail with a plain (non- + // ServiceError) error, exercising the generic 500 fallback. + sqlDB, err := db.DB() + Expect(err).NotTo(HaveOccurred()) + Expect(sqlDB.Close()).To(Succeed()) + + req := httptest.NewRequest(http.MethodGet, "/agents", nil) + rec := httptest.NewRecorder() + + router.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusInternalServerError)) + Expect(rec.Header().Get("Content-Type")).To(Equal("application/problem+json")) + + var body server.Error + Expect(json.NewDecoder(rec.Body).Decode(&body)).To(Succeed()) + Expect(body.Type).To(Equal("list-error")) + Expect(body.Status).NotTo(BeNil()) + Expect(*body.Status).To(Equal(500)) + }) + }) + + Describe("PUT /agents/{agentId}/heartbeat", func() { + It("returns 200 on success", func() { + body := `{"name":"hb-agent","topic_name":"dcm.agent.hb-agent","service_types":["vm"]}` + reqCreate := httptest.NewRequest(http.MethodPost, "/agents", strings.NewReader(body)) + reqCreate.Header.Set("Content-Type", "application/json") + recCreate := httptest.NewRecorder() + router.ServeHTTP(recCreate, reqCreate) + Expect(recCreate.Code).To(Equal(http.StatusCreated)) + + var created api.Agent + Expect(json.NewDecoder(recCreate.Body).Decode(&created)).To(Succeed()) + + hbBody := `{"consumer_lag":0,"timestamp":"2026-07-31T10:00:00Z"}` + reqHb := httptest.NewRequest(http.MethodPut, "/agents/"+*created.AgentId+"/heartbeat", strings.NewReader(hbBody)) + reqHb.Header.Set("Content-Type", "application/json") + recHb := httptest.NewRecorder() + router.ServeHTTP(recHb, reqHb) + + Expect(recHb.Code).To(Equal(http.StatusOK)) + }) + + It("returns 404 on unknown agent", func() { + hbBody := `{"consumer_lag":0,"timestamp":"2026-07-31T10:00:00Z"}` + req := httptest.NewRequest(http.MethodPut, "/agents/"+uuid.New().String()+"/heartbeat", strings.NewReader(hbBody)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + + router.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusNotFound)) + }) + }) +}) diff --git a/internal/sp/healthcheck/healthcheck_suite_test.go b/internal/agent/healthcheck/healthcheck_suite_test.go similarity index 63% rename from internal/sp/healthcheck/healthcheck_suite_test.go rename to internal/agent/healthcheck/healthcheck_suite_test.go index 510e24f..2a0cb5e 100644 --- a/internal/sp/healthcheck/healthcheck_suite_test.go +++ b/internal/agent/healthcheck/healthcheck_suite_test.go @@ -7,7 +7,7 @@ import ( . "github.com/onsi/gomega" ) -func TestHealthcheck(t *testing.T) { +func TestHealthCheck(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Healthcheck Suite") + RunSpecs(t, "Agent Health Monitor Suite") } diff --git a/internal/agent/healthcheck/monitor.go b/internal/agent/healthcheck/monitor.go new file mode 100644 index 0000000..69ec710 --- /dev/null +++ b/internal/agent/healthcheck/monitor.go @@ -0,0 +1,67 @@ +// Package healthcheck monitors agent heartbeats and marks agents unavailable. +package healthcheck + +import ( + "context" + "log/slog" + "sync" + "time" + + agentstore "github.com/dcm-project/control-plane/internal/agent/store/agent" +) + +type Monitor struct { + store agentstore.Agent + heartbeatTimeout time.Duration + interval time.Duration + stopCh chan struct{} + stopOnce sync.Once + wg sync.WaitGroup +} + +func NewMonitor(store agentstore.Agent, heartbeatTimeout, interval time.Duration) *Monitor { + return &Monitor{ + store: store, + heartbeatTimeout: heartbeatTimeout, + interval: interval, + stopCh: make(chan struct{}), + } +} + +func (m *Monitor) Start(ctx context.Context) { + m.wg.Add(1) + go func() { + defer m.wg.Done() + m.sweep(ctx) + + ticker := time.NewTicker(m.interval) + defer ticker.Stop() + for { + select { + case <-m.stopCh: + return + case <-ctx.Done(): + return + case <-ticker.C: + m.sweep(ctx) + } + } + }() +} + +func (m *Monitor) Stop() { + m.stopOnce.Do(func() { close(m.stopCh) }) + m.wg.Wait() +} + +// sweep flips stale agents to Unavailable via a single atomic conditional +// UPDATE (MarkStaleUnavailable) instead of listing agents and then writing +// per-agent based on that earlier snapshot: a read-then-write here would +// leave a window in which a heartbeat landing between the read and the +// write gets clobbered back to Unavailable by this sweep. +func (m *Monitor) sweep(ctx context.Context) { + cutoff := time.Now().Add(-m.heartbeatTimeout) + if err := m.store.MarkStaleUnavailable(ctx, cutoff); err != nil { + slog.Error("health monitor: failed to mark stale agents unavailable", "error", err) + } +} diff --git a/internal/agent/healthcheck/monitor_test.go b/internal/agent/healthcheck/monitor_test.go new file mode 100644 index 0000000..d45c7e1 --- /dev/null +++ b/internal/agent/healthcheck/monitor_test.go @@ -0,0 +1,144 @@ +package healthcheck_test + +import ( + "context" + "time" + + "github.com/dcm-project/control-plane/internal/agent/healthcheck" + "github.com/dcm-project/control-plane/internal/agent/store/model" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + agentstore "github.com/dcm-project/control-plane/internal/agent/store/agent" +) + +var _ = Describe("Health Monitor", func() { + var ( + db *gorm.DB + store agentstore.Agent + monitor *healthcheck.Monitor + ctx context.Context + ) + + BeforeEach(func() { + var err error + db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + Expect(err).NotTo(HaveOccurred()) + // sqlite's ":memory:" DSN gives each new physical connection its own + // empty database, so once the monitor's background sweep goroutine + // and the test's own Eventually/Consistently assertions query + // concurrently, a second pooled connection would see "no such + // table" instead of the migrated schema. + sqlDB, err := db.DB() + Expect(err).NotTo(HaveOccurred()) + sqlDB.SetMaxOpenConns(1) + Expect(db.AutoMigrate(&model.Agent{})).To(Succeed()) + + store = agentstore.NewAgent(db) + monitor = healthcheck.NewMonitor(store, 30*time.Second, 5*time.Second) + ctx = context.Background() + }) + + AfterEach(func() { + sqlDB, _ := db.DB() + _ = sqlDB.Close() + }) + + It("marks agent Unavailable when heartbeat expires", func() { + expiredHB := time.Now().Add(-1 * time.Minute) + a := model.Agent{ + ID: uuid.New().String(), + Name: "expired-agent", + TopicName: "dcm.agent.expired-agent", + HealthStatus: model.AgentHealthStatusReady, + LastHeartbeat: &expiredHB, + } + _, err := store.Create(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + monitor.Start(ctx) + defer monitor.Stop() + + Eventually(func() model.AgentHealthStatus { + updated, err := store.Get(ctx, a.ID) + Expect(err).NotTo(HaveOccurred()) + return updated.HealthStatus + }).WithTimeout(2 * time.Second).WithPolling(20 * time.Millisecond).Should(Equal(model.AgentHealthStatusUnavailable)) + }) + + It("uses created_at as grace period for new agents without heartbeat", func() { + a := model.Agent{ + ID: uuid.New().String(), + Name: "new-agent", + TopicName: "dcm.agent.new-agent", + HealthStatus: model.AgentHealthStatusReady, + } + _, err := store.Create(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + monitor.Start(ctx) + defer monitor.Stop() + + Consistently(func() model.AgentHealthStatus { + updated, err := store.Get(ctx, a.ID) + Expect(err).NotTo(HaveOccurred()) + return updated.HealthStatus + }, 200*time.Millisecond, 20*time.Millisecond).Should(Equal(model.AgentHealthStatusReady)) + }) + + It("does not mark already-unavailable agents", func() { + expiredHB := time.Now().Add(-1 * time.Minute) + a := model.Agent{ + ID: uuid.New().String(), + Name: "already-unavailable", + TopicName: "dcm.agent.already-unavailable", + HealthStatus: model.AgentHealthStatusUnavailable, + LastHeartbeat: &expiredHB, + } + _, err := store.Create(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + monitor.Start(ctx) + defer monitor.Stop() + + Consistently(func() model.AgentHealthStatus { + updated, err := store.Get(ctx, a.ID) + Expect(err).NotTo(HaveOccurred()) + return updated.HealthStatus + }, 200*time.Millisecond, 20*time.Millisecond).Should(Equal(model.AgentHealthStatusUnavailable)) + }) + + It("re-registration clears Unavailable", func() { + expiredHB := time.Now().Add(-1 * time.Minute) + a := model.Agent{ + ID: uuid.New().String(), + Name: "re-reg-agent", + TopicName: "dcm.agent.re-reg-agent", + HealthStatus: model.AgentHealthStatusUnavailable, + LastHeartbeat: &expiredHB, + } + _, err := store.Create(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + // Update's health_status/last_heartbeat write is CAS-guarded by the + // same monotonicity rule as UpdateHeartbeatIfNewer, so - matching + // how the real caller (RegisterOrUpdate) always supplies a fresh + // time.Now() - the re-registration must carry a newer timestamp + // than what's already stored for it to actually clear Unavailable. + freshHB := time.Now() + a.HealthStatus = model.AgentHealthStatusReady + a.LastHeartbeat = &freshHB + _, err = store.Update(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + updated, err := store.Get(ctx, a.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(updated.HealthStatus).To(Equal(model.AgentHealthStatusReady)) + }) +}) diff --git a/internal/agent/service/agent.go b/internal/agent/service/agent.go new file mode 100644 index 0000000..53fb33b --- /dev/null +++ b/internal/agent/service/agent.go @@ -0,0 +1,174 @@ +// Package service implements agent business logic. +package service + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + api "github.com/dcm-project/control-plane/api/agent/v1alpha1" + agentstore "github.com/dcm-project/control-plane/internal/agent/store/agent" + "github.com/dcm-project/control-plane/internal/agent/store/model" + "github.com/google/uuid" +) + +type AgentService struct { + store agentstore.Agent + consumerLagThreshold int64 +} + +func NewAgentService(store agentstore.Agent, consumerLagThreshold int64) *AgentService { + return &AgentService{ + store: store, + consumerLagThreshold: consumerLagThreshold, + } +} + +func (s *AgentService) RegisterOrUpdate(ctx context.Context, req api.AgentRegistrationRequest) (*api.Agent, bool, error) { + if !strings.HasPrefix(req.TopicName, "dcm.agent.") { + return nil, false, NewValidationError("topic_name must start with 'dcm.agent.'") + } + + existing, err := s.store.GetByName(ctx, req.Name) + if err != nil && !errors.Is(err, agentstore.ErrAgentNotFound) { + return nil, false, err + } + + cost := model.AgentCost(req.Cost) + + if existing != nil { + existing.Environment = req.Environment + existing.ServiceTypes = req.ServiceTypes + existing.Cost = &cost + existing.TopicName = req.TopicName + existing.HealthStatus = model.AgentHealthStatusReady + now := time.Now() + existing.LastHeartbeat = &now + + updated, err := s.store.Update(ctx, *existing) + if err != nil { + if errors.Is(err, agentstore.ErrAgentConflict) { + return nil, false, NewConflictError(fmt.Sprintf("topic name %q is already registered to another agent", req.TopicName)) + } + return nil, false, err + } + result := modelToAPI(updated) + return &result, false, nil + } + + agent := model.Agent{ + ID: uuid.New().String(), + Name: req.Name, + Environment: req.Environment, + ServiceTypes: req.ServiceTypes, + Cost: &cost, + TopicName: req.TopicName, + HealthStatus: model.AgentHealthStatusReady, + } + + created, err := s.store.Create(ctx, agent) + if err != nil { + if errors.Is(err, agentstore.ErrAgentConflict) { + return nil, false, NewConflictError(fmt.Sprintf("agent name %q or topic name %q is already registered", req.Name, req.TopicName)) + } + return nil, false, err + } + result := modelToAPI(created) + return &result, true, nil +} + +func (s *AgentService) Get(ctx context.Context, id string) (*api.Agent, error) { + agent, err := s.store.Get(ctx, id) + if err != nil { + if errors.Is(err, agentstore.ErrAgentNotFound) { + return nil, NewNotFoundError("agent not found") + } + return nil, err + } + result := modelToAPI(agent) + return &result, nil +} + +func (s *AgentService) List(ctx context.Context, healthStatus string, pageSize int, pageToken string) (*AgentListResult, error) { + var filter *agentstore.AgentFilter + if healthStatus != "" { + hs := model.AgentHealthStatus(healthStatus) + filter = &agentstore.AgentFilter{HealthStatus: &hs} + } + + pagination := &agentstore.Pagination{Limit: pageSize, PageToken: pageToken} + + storeResult, err := s.store.List(ctx, filter, pagination) + if err != nil { + if errors.Is(err, agentstore.ErrInvalidPageToken) { + return nil, NewValidationError("invalid page_token") + } + return nil, err + } + + result := &AgentListResult{NextPageToken: storeResult.NextPageToken} + for _, a := range storeResult.Agents { + result.Agents = append(result.Agents, modelToAPI(&a)) + } + return result, nil +} + +func (s *AgentService) Heartbeat(ctx context.Context, agentID string, req api.HeartbeatRequest) (*api.Agent, error) { + if _, err := s.store.Get(ctx, agentID); err != nil { + if errors.Is(err, agentstore.ErrAgentNotFound) { + return nil, NewNotFoundError("agent not found") + } + return nil, err + } + + healthStatus := model.AgentHealthStatusReady + if req.ConsumerLag >= s.consumerLagThreshold { + healthStatus = model.AgentHealthStatusCongested + } + + // Atomic conditional UPDATE, not read-compare-then-write: two concurrent + // out-of-order heartbeats could otherwise both pass a Go-level staleness + // check and let the later Update() win regardless of timestamp order. + if _, err := s.store.UpdateHeartbeatIfNewer(ctx, agentID, req.Timestamp, healthStatus); err != nil { + return nil, err + } + // Re-fetch rather than reuse the pre-update snapshot: a concurrent + // heartbeat or health sweep could have changed the row in between. + updated, err := s.store.Get(ctx, agentID) + if err != nil { + return nil, err + } + result := modelToAPI(updated) + return &result, nil +} + +type AgentListResult struct { + Agents []api.Agent + NextPageToken string +} + +func modelToAPI(m *model.Agent) api.Agent { + hs := api.AgentHealthStatus(m.HealthStatus) + st := make([]string, len(m.ServiceTypes)) + copy(st, m.ServiceTypes) + a := api.Agent{ + AgentId: &m.ID, + Name: &m.Name, + Environment: &m.Environment, + ServiceTypes: &st, + TopicName: &m.TopicName, + HealthStatus: &hs, + CreateTime: &m.CreateTime, + UpdateTime: &m.UpdateTime, + } + if m.LastHeartbeat != nil { + a.LastHeartbeat = m.LastHeartbeat + } + if m.Cost != nil { + cost := api.AgentCost(*m.Cost) + a.Cost = &cost + } + return a +} diff --git a/internal/agent/service/agent_suite_test.go b/internal/agent/service/agent_suite_test.go new file mode 100644 index 0000000..9c1ea27 --- /dev/null +++ b/internal/agent/service/agent_suite_test.go @@ -0,0 +1,13 @@ +package service_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestAgentService(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Agent Service Suite") +} diff --git a/internal/agent/service/agent_test.go b/internal/agent/service/agent_test.go new file mode 100644 index 0000000..8964f80 --- /dev/null +++ b/internal/agent/service/agent_test.go @@ -0,0 +1,253 @@ +package service_test + +import ( + "context" + "time" + + api "github.com/dcm-project/control-plane/api/agent/v1alpha1" + "github.com/dcm-project/control-plane/internal/agent/service" + "github.com/dcm-project/control-plane/internal/agent/store/model" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + agentstore "github.com/dcm-project/control-plane/internal/agent/store/agent" +) + +var _ = Describe("AgentService", func() { + var ( + db *gorm.DB + store agentstore.Agent + svc *service.AgentService + ctx context.Context + defaultLag int64 + ) + + BeforeEach(func() { + var err error + db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(db.AutoMigrate(&model.Agent{})).To(Succeed()) + + store = agentstore.NewAgent(db) + defaultLag = 100 + svc = service.NewAgentService(store, defaultLag) + ctx = context.Background() + }) + + AfterEach(func() { + sqlDB, _ := db.DB() + _ = sqlDB.Close() + }) + + Describe("RegisterOrUpdate", func() { + It("creates new agent with valid payload", func() { + req := validRegistrationRequest("new-agent") + + result, created, err := svc.RegisterOrUpdate(ctx, req) + + Expect(err).NotTo(HaveOccurred()) + Expect(created).To(BeTrue()) + Expect(result).NotTo(BeNil()) + Expect(result.AgentId).NotTo(BeNil()) + Expect(*result.Name).To(Equal("new-agent")) + }) + + It("returns 200 and updates existing agent by name", func() { + req := validRegistrationRequest("existing-agent") + _, _, err := svc.RegisterOrUpdate(ctx, req) + Expect(err).NotTo(HaveOccurred()) + + req.Environment = "staging" + result, created, err := svc.RegisterOrUpdate(ctx, req) + + Expect(err).NotTo(HaveOccurred()) + Expect(created).To(BeFalse()) + Expect(result).NotTo(BeNil()) + }) + + It("round-trips the cost enum value on create and re-registration", func() { + req := validRegistrationRequest("cost-agent") + req.Cost = api.AgentRegistrationRequestCostHigh + + result, _, err := svc.RegisterOrUpdate(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Cost).NotTo(BeNil()) + Expect(*result.Cost).To(Equal(api.AgentCostHigh)) + + req.Cost = api.AgentRegistrationRequestCostLow + result, _, err = svc.RegisterOrUpdate(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Cost).NotTo(BeNil()) + Expect(*result.Cost).To(Equal(api.AgentCostLow)) + }) + + It("allows empty service_types on re-registration", func() { + req := validRegistrationRequest("re-reg-agent") + _, _, err := svc.RegisterOrUpdate(ctx, req) + Expect(err).NotTo(HaveOccurred()) + + req.ServiceTypes = []string{} + _, _, err = svc.RegisterOrUpdate(ctx, req) + + Expect(err).NotTo(HaveOccurred()) + }) + + It("rejects topic_name not starting with dcm.agent.", func() { + req := validRegistrationRequest("bad-topic") + req.TopicName = "invalid.topic.name" + + _, _, err := svc.RegisterOrUpdate(ctx, req) + + Expect(err).To(HaveOccurred()) + svcErr, ok := err.(*service.ServiceError) + Expect(ok).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeValidation)) + }) + + It("returns a conflict error when a different agent already owns the topic_name (K)", func() { + first := validRegistrationRequest("topic-owner") + _, _, err := svc.RegisterOrUpdate(ctx, first) + Expect(err).NotTo(HaveOccurred()) + + second := validRegistrationRequest("topic-squatter") + second.TopicName = first.TopicName + + _, _, err = svc.RegisterOrUpdate(ctx, second) + + Expect(err).To(HaveOccurred()) + svcErr, ok := err.(*service.ServiceError) + Expect(ok).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeConflict)) + }) + }) + + Describe("Heartbeat", func() { + It("marks Ready when consumer_lag < threshold", func() { + req := validRegistrationRequest("ready-agent") + result, _, err := svc.RegisterOrUpdate(ctx, req) + Expect(err).NotTo(HaveOccurred()) + + now := time.Now() + hbReq := api.HeartbeatRequest{ + ConsumerLag: 0, + Timestamp: now, + } + + updated, err := svc.Heartbeat(ctx, *result.AgentId, hbReq) + + Expect(err).NotTo(HaveOccurred()) + Expect(updated).NotTo(BeNil()) + Expect(*updated.HealthStatus).To(Equal(api.AgentHealthStatus("ready"))) + }) + + It("marks Congested when consumer_lag >= threshold", func() { + req := validRegistrationRequest("congested-agent") + result, _, err := svc.RegisterOrUpdate(ctx, req) + Expect(err).NotTo(HaveOccurred()) + + now := time.Now() + hbReq := api.HeartbeatRequest{ + ConsumerLag: defaultLag + 1, + Timestamp: now, + } + + updated, err := svc.Heartbeat(ctx, *result.AgentId, hbReq) + + Expect(err).NotTo(HaveOccurred()) + Expect(updated).NotTo(BeNil()) + Expect(*updated.HealthStatus).To(Equal(api.AgentHealthStatus("congested"))) + }) + + It("ignores stale heartbeat timestamp without overwriting the newer one (L)", func() { + req := validRegistrationRequest("stale-hb-agent") + result, _, err := svc.RegisterOrUpdate(ctx, req) + Expect(err).NotTo(HaveOccurred()) + + now := time.Now() + hbReq := api.HeartbeatRequest{ + ConsumerLag: 0, + Timestamp: now, + } + _, err = svc.Heartbeat(ctx, *result.AgentId, hbReq) + Expect(err).NotTo(HaveOccurred()) + + staleReq := api.HeartbeatRequest{ + ConsumerLag: 0, + Timestamp: now.Add(-1 * time.Hour), + } + updated, err := svc.Heartbeat(ctx, *result.AgentId, staleReq) + + Expect(err).NotTo(HaveOccurred()) + // The atomic CAS (UpdateHeartbeatIfNewer) must leave the + // previously-recorded, newer timestamp in place rather than + // letting the stale write land. + Expect(updated.LastHeartbeat).NotTo(BeNil()) + Expect(*updated.LastHeartbeat).To(BeTemporally("~", now, time.Second)) + }) + + It("returns not-found for unknown agent", func() { + hbReq := api.HeartbeatRequest{ + ConsumerLag: 0, + Timestamp: time.Now(), + } + + _, err := svc.Heartbeat(ctx, uuid.New().String(), hbReq) + + Expect(err).To(HaveOccurred()) + svcErr, ok := err.(*service.ServiceError) + Expect(ok).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeNotFound)) + }) + }) + + Describe("List", func() { + It("paginates through all agents via the returned NextPageToken", func() { + names := []string{"list-agent-1", "list-agent-2", "list-agent-3"} + for _, n := range names { + _, _, err := svc.RegisterOrUpdate(ctx, validRegistrationRequest(n)) + Expect(err).NotTo(HaveOccurred()) + } + + first, err := svc.List(ctx, "", 2, "") + Expect(err).NotTo(HaveOccurred()) + Expect(first.Agents).To(HaveLen(2)) + Expect(first.NextPageToken).NotTo(BeEmpty()) + + second, err := svc.List(ctx, "", 2, first.NextPageToken) + Expect(err).NotTo(HaveOccurred()) + Expect(second.Agents).To(HaveLen(1)) + Expect(second.NextPageToken).To(BeEmpty()) + + seen := make([]string, 0, len(first.Agents)+len(second.Agents)) + for _, a := range append(first.Agents, second.Agents...) { + seen = append(seen, *a.Name) + } + Expect(seen).To(Equal(names)) + }) + + It("returns a validation error for a malformed page_token", func() { + _, err := svc.List(ctx, "", 10, "not-valid-base64!!!") + + Expect(err).To(HaveOccurred()) + svcErr, ok := err.(*service.ServiceError) + Expect(ok).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeValidation)) + }) + }) +}) + +func validRegistrationRequest(name string) api.AgentRegistrationRequest { + return api.AgentRegistrationRequest{ + Name: name, + TopicName: "dcm.agent." + name, + ServiceTypes: []string{"vm", "container"}, + Environment: "production", + Cost: api.AgentRegistrationRequestCostMedium, + } +} diff --git a/internal/agent/service/errors.go b/internal/agent/service/errors.go new file mode 100644 index 0000000..0da8802 --- /dev/null +++ b/internal/agent/service/errors.go @@ -0,0 +1,59 @@ +package service + +import ( + "errors" + "fmt" +) + +type ErrorCode string + +// Error codes are RFC 7807 "type" URIs, matching the convention used by +// internal/sp/service/errors.go, so the agent and SP domains don't diverge +// on error identifier format. +const ( + ErrCodeNotFound ErrorCode = "https://dcm.example.com/errors/not-found" + ErrCodeValidation ErrorCode = "https://dcm.example.com/errors/validation" + ErrCodeConflict ErrorCode = "https://dcm.example.com/errors/conflict" + ErrCodeNotImplemented ErrorCode = "https://dcm.example.com/errors/not-implemented" +) + +type ServiceError struct { + Code ErrorCode + Message string +} + +func (e *ServiceError) Error() string { + return fmt.Sprintf("%s: %s", e.Code, e.Message) +} + +func NewNotFoundError(msg string) *ServiceError { + return &ServiceError{Code: ErrCodeNotFound, Message: msg} +} + +func NewValidationError(msg string) *ServiceError { + return &ServiceError{Code: ErrCodeValidation, Message: msg} +} + +func NewConflictError(msg string) *ServiceError { + return &ServiceError{Code: ErrCodeConflict, Message: msg} +} + +func NewNotImplementedError() *ServiceError { + return &ServiceError{Code: ErrCodeNotImplemented, Message: "not implemented"} +} + +// IsClientError returns true if err is a ServiceError representing a +// client-side (4xx) problem, mirroring internal/sp/service.IsClientError. +// ErrCodeNotImplemented is deliberately excluded: it's a server-side gap +// (501), not something the caller did wrong. If svcErr is non-nil it is +// populated with the unwrapped error. +func IsClientError(err error, svcErr **ServiceError) bool { + if !errors.As(err, svcErr) { + return false + } + switch (*svcErr).Code { + case ErrCodeValidation, ErrCodeNotFound, ErrCodeConflict: + return true + } + return false +} diff --git a/internal/agent/store/agent/agent.go b/internal/agent/store/agent/agent.go new file mode 100644 index 0000000..8300b89 --- /dev/null +++ b/internal/agent/store/agent/agent.go @@ -0,0 +1,251 @@ +// Package agent provides the GORM-based store for Agent entities. +package agent + +import ( + "context" + "encoding/json" + "errors" + "strings" + "time" + + "github.com/dcm-project/control-plane/internal/agent/store/model" + "gorm.io/gorm" +) + +var ( + ErrAgentNotFound = errors.New("agent not found") + // ErrAgentConflict is returned when a Create/Update violates the unique + // constraint on name or topic_name (e.g. two concurrent registrations of + // a new agent with the same name). + ErrAgentConflict = errors.New("agent name or topic name already in use") +) + +// isUniqueViolation detects a unique-constraint violation across the DB +// drivers this store runs against (Postgres in production, SQLite in +// tests), matching the pattern already used in internal/auth/service and +// internal/placement/store for the same purpose. +func isUniqueViolation(err error) bool { + if errors.Is(err, gorm.ErrDuplicatedKey) { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "unique constraint") || strings.Contains(msg, "duplicate key") +} + +type AgentFilter struct { + HealthStatus *model.AgentHealthStatus +} + +// Pagination carries the requested page size and an opaque page_token (as +// previously returned in AgentListResult.NextPageToken) for List. A zero +// Limit falls back to defaultListPageSize. +type Pagination struct { + Limit int + PageToken string +} + +// AgentListResult is the result of a List call: the page of agents plus an +// opaque NextPageToken (empty when there are no further pages), mirroring +// internal/policy/store/policy.go's PolicyListResult. +type AgentListResult struct { + Agents model.AgentList + NextPageToken string +} + +type Agent interface { + Create(ctx context.Context, agent model.Agent) (*model.Agent, error) + Get(ctx context.Context, id string) (*model.Agent, error) + GetByName(ctx context.Context, name string) (*model.Agent, error) + // List returns agents matching filter, paginated per pagination. Pass + // pagination.PageToken from a prior AgentListResult.NextPageToken to + // fetch the next page; ErrInvalidPageToken is returned if it can't be + // decoded. + List(ctx context.Context, filter *AgentFilter, pagination *Pagination) (*AgentListResult, error) + Update(ctx context.Context, agent model.Agent) (*model.Agent, error) + Delete(ctx context.Context, id string) error + ListReady(ctx context.Context) (model.AgentList, error) + // UpdateHeartbeatIfNewer atomically updates last_heartbeat/health_status + // only if ts is strictly newer than the stored last_heartbeat (or none is + // stored yet). Returns applied=false without error if ts is stale + // (monotonicity), so two concurrent/out-of-order heartbeats can't race + // past each other's in-memory read and have the older one win. + UpdateHeartbeatIfNewer(ctx context.Context, id string, ts time.Time, healthStatus model.AgentHealthStatus) (bool, error) + // MarkStaleUnavailable flips every agent whose heartbeat (or, absent + // one, creation time) is older than cutoff to Unavailable in a single + // conditional UPDATE, so the health check has no read-then-write window + // in which a concurrent heartbeat could be clobbered back to Unavailable. + MarkStaleUnavailable(ctx context.Context, cutoff time.Time) error +} + +type AgentStore struct { + db *gorm.DB +} + +var _ Agent = (*AgentStore)(nil) + +func NewAgent(db *gorm.DB) Agent { + return &AgentStore{db: db} +} + +func (s *AgentStore) Create(ctx context.Context, agent model.Agent) (*model.Agent, error) { + if err := s.db.WithContext(ctx).Create(&agent).Error; err != nil { + if isUniqueViolation(err) { + return nil, ErrAgentConflict + } + return nil, err + } + return &agent, nil +} + +func (s *AgentStore) Get(ctx context.Context, id string) (*model.Agent, error) { + var agent model.Agent + if err := s.db.WithContext(ctx).First(&agent, "id = ?", id).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrAgentNotFound + } + return nil, err + } + return &agent, nil +} + +func (s *AgentStore) GetByName(ctx context.Context, name string) (*model.Agent, error) { + var agent model.Agent + if err := s.db.WithContext(ctx).First(&agent, "name = ?", name).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, ErrAgentNotFound + } + return nil, err + } + return &agent, nil +} + +// defaultListPageSize is used when pagination is nil or its Limit is <= 0, +// matching the max_page_size default documented in +// api/agent/v1alpha1/openapi.yaml. +const defaultListPageSize = 100 + +func (s *AgentStore) List(ctx context.Context, filter *AgentFilter, pagination *Pagination) (*AgentListResult, error) { + var agents model.AgentList + query := s.db.WithContext(ctx) + + if filter != nil && filter.HealthStatus != nil { + query = query.Where("health_status = ?", *filter.HealthStatus) + } + + pageSize := defaultListPageSize + offset := 0 + if pagination != nil { + if pagination.Limit > 0 { + pageSize = pagination.Limit + } + if pagination.PageToken != "" { + var err error + offset, err = decodePageToken(pagination.PageToken) + if err != nil { + return nil, err + } + } + } + + // Query with limit+1 to detect whether there are more results beyond + // this page, mirroring internal/policy/store/policy.go's List. + query = query.Order("name ASC").Limit(pageSize + 1).Offset(offset) + if err := query.Find(&agents).Error; err != nil { + return nil, err + } + + result := &AgentListResult{Agents: agents} + if len(agents) > pageSize { + result.Agents = agents[:pageSize] + nextOffset := offset + pageSize + nextToken, err := encodePageToken(nextOffset) + if err != nil { + return nil, err + } + result.NextPageToken = nextToken + } + return result, nil +} + +func (s *AgentStore) Update(ctx context.Context, agent model.Agent) (*model.Agent, error) { + // service_types has a `serializer:json` tag for struct-based Create/Save, + // but a map-based Updates call bypasses that serializer, so it must be + // marshalled by hand here or the driver receives a raw []string and + // mis-binds it as a composite value. + serviceTypesJSON, err := json.Marshal(agent.ServiceTypes) + if err != nil { + return nil, err + } + + result := s.db.WithContext(ctx).Model(&model.Agent{}).Where("id = ?", agent.ID).Updates(map[string]any{ + "environment": agent.Environment, + "service_types": string(serviceTypesJSON), + "cost": agent.Cost, + "topic_name": agent.TopicName, + }) + if result.Error != nil { + if isUniqueViolation(result.Error) { + return nil, ErrAgentConflict + } + return nil, result.Error + } + if result.RowsAffected == 0 { + return nil, ErrAgentNotFound + } + + // CAS-guarded by the same monotonicity rule as UpdateHeartbeatIfNewer, so + // a concurrent, newer heartbeat can't be clobbered by this re-registration. + if agent.LastHeartbeat != nil { + if err := s.db.WithContext(ctx).Model(&model.Agent{}). + Where("id = ? AND (last_heartbeat IS NULL OR last_heartbeat < ?)", agent.ID, *agent.LastHeartbeat). + Updates(map[string]any{ + "last_heartbeat": agent.LastHeartbeat, + "health_status": agent.HealthStatus, + }).Error; err != nil { + return nil, err + } + } + + return s.Get(ctx, agent.ID) +} + +func (s *AgentStore) Delete(ctx context.Context, id string) error { + result := s.db.WithContext(ctx).Where("id = ?", id).Delete(&model.Agent{}) + if result.Error != nil { + return result.Error + } + if result.RowsAffected == 0 { + return ErrAgentNotFound + } + return nil +} + +func (s *AgentStore) ListReady(ctx context.Context) (model.AgentList, error) { + var agents model.AgentList + if err := s.db.WithContext(ctx). + Where("health_status = ?", model.AgentHealthStatusReady). + Find(&agents).Error; err != nil { + return nil, err + } + return agents, nil +} + +func (s *AgentStore) UpdateHeartbeatIfNewer(ctx context.Context, id string, ts time.Time, healthStatus model.AgentHealthStatus) (bool, error) { + result := s.db.WithContext(ctx).Model(&model.Agent{}). + Where("id = ? AND (last_heartbeat IS NULL OR last_heartbeat < ?)", id, ts). + Updates(map[string]any{ + "last_heartbeat": ts, + "health_status": healthStatus, + }) + if result.Error != nil { + return false, result.Error + } + return result.RowsAffected > 0, nil +} + +func (s *AgentStore) MarkStaleUnavailable(ctx context.Context, cutoff time.Time) error { + return s.db.WithContext(ctx).Model(&model.Agent{}). + Where("health_status != ?", model.AgentHealthStatusUnavailable). + Where("(last_heartbeat IS NOT NULL AND last_heartbeat < ?) OR (last_heartbeat IS NULL AND create_time < ?)", cutoff, cutoff). + Update("health_status", model.AgentHealthStatusUnavailable).Error +} diff --git a/internal/sp/handlers/provider/handler_suite_test.go b/internal/agent/store/agent/agent_suite_test.go similarity index 55% rename from internal/sp/handlers/provider/handler_suite_test.go rename to internal/agent/store/agent/agent_suite_test.go index 1653efe..6913cf9 100644 --- a/internal/sp/handlers/provider/handler_suite_test.go +++ b/internal/agent/store/agent/agent_suite_test.go @@ -1,4 +1,4 @@ -package provider_test +package agent_test import ( "testing" @@ -7,7 +7,7 @@ import ( . "github.com/onsi/gomega" ) -func TestHandlers(t *testing.T) { +func TestAgentStore(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Handlers Suite") + RunSpecs(t, "Agent Store Suite") } diff --git a/internal/agent/store/agent/agent_test.go b/internal/agent/store/agent/agent_test.go new file mode 100644 index 0000000..8f637c2 --- /dev/null +++ b/internal/agent/store/agent/agent_test.go @@ -0,0 +1,398 @@ +package agent_test + +import ( + "context" + "time" + + "github.com/dcm-project/control-plane/internal/agent/store/agent" + "github.com/dcm-project/control-plane/internal/agent/store/model" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +var _ = Describe("Agent Store", func() { + var ( + db *gorm.DB + agentStore agent.Agent + ctx context.Context + ) + + BeforeEach(func() { + var err error + db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(db.AutoMigrate(&model.Agent{})).To(Succeed()) + + agentStore = agent.NewAgent(db) + ctx = context.Background() + }) + + AfterEach(func() { + sqlDB, _ := db.DB() + _ = sqlDB.Close() + }) + + Describe("Create", func() { + It("persists the agent and returns generated ID", func() { + a := newAgent("create-test") + created, err := agentStore.Create(ctx, a) + + Expect(err).NotTo(HaveOccurred()) + Expect(created.ID).To(Equal(a.ID)) + Expect(created.Name).To(Equal("create-test")) + Expect(created.TopicName).To(Equal("dcm.agent.create-test")) + Expect(created.HealthStatus).To(Equal(model.AgentHealthStatusReady)) + }) + + It("rejects duplicate names with ErrAgentConflict (K)", func() { + a1 := newAgent("duplicate-name") + _, err := agentStore.Create(ctx, a1) + Expect(err).NotTo(HaveOccurred()) + + a2 := newAgent("duplicate-name") + _, err = agentStore.Create(ctx, a2) + Expect(err).To(MatchError(agent.ErrAgentConflict)) + }) + + It("rejects duplicate topic_name across different agent names with ErrAgentConflict (K)", func() { + a1 := newAgent("agent-one") + _, err := agentStore.Create(ctx, a1) + Expect(err).NotTo(HaveOccurred()) + + a2 := newAgent("agent-two") + a2.TopicName = a1.TopicName + _, err = agentStore.Create(ctx, a2) + Expect(err).To(MatchError(agent.ErrAgentConflict)) + }) + }) + + Describe("Get", func() { + It("retrieves by ID", func() { + a := newAgent("get-test") + _, err := agentStore.Create(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + found, err := agentStore.Get(ctx, a.ID) + + Expect(err).NotTo(HaveOccurred()) + Expect(found.Name).To(Equal("get-test")) + }) + + It("returns ErrAgentNotFound for missing ID", func() { + _, err := agentStore.Get(ctx, uuid.New().String()) + + Expect(err).To(Equal(agent.ErrAgentNotFound)) + }) + }) + + Describe("GetByName", func() { + It("retrieves by name", func() { + a := newAgent("named-agent") + _, err := agentStore.Create(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + found, err := agentStore.GetByName(ctx, "named-agent") + + Expect(err).NotTo(HaveOccurred()) + Expect(found.ID).To(Equal(a.ID)) + }) + + It("returns ErrAgentNotFound for missing name", func() { + _, err := agentStore.GetByName(ctx, "non-existent") + + Expect(err).To(Equal(agent.ErrAgentNotFound)) + }) + }) + + Describe("List", func() { + It("returns all agents when filter is nil", func() { + _, err := agentStore.Create(ctx, newAgent("a1")) + Expect(err).NotTo(HaveOccurred()) + _, err = agentStore.Create(ctx, newAgent("a2")) + Expect(err).NotTo(HaveOccurred()) + + result, err := agentStore.List(ctx, nil, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(result.Agents).To(HaveLen(2)) + Expect(result.NextPageToken).To(BeEmpty()) + }) + + It("filters by health_status", func() { + a1 := newAgent("ready-agent") + _, err := agentStore.Create(ctx, a1) + Expect(err).NotTo(HaveOccurred()) + + a2 := newAgent("unavailable-agent") + a2.HealthStatus = model.AgentHealthStatusUnavailable + _, err = agentStore.Create(ctx, a2) + Expect(err).NotTo(HaveOccurred()) + + readyStatus := model.AgentHealthStatusReady + result, err := agentStore.List(ctx, &agent.AgentFilter{HealthStatus: &readyStatus}, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(result.Agents).To(HaveLen(1)) + Expect(result.Agents[0].Name).To(Equal("ready-agent")) + }) + + It("respects the requested page size", func() { + for i := 0; i < 3; i++ { + _, err := agentStore.Create(ctx, newAgent(uuid.New().String())) + Expect(err).NotTo(HaveOccurred()) + } + + result, err := agentStore.List(ctx, nil, &agent.Pagination{Limit: 2}) + + Expect(err).NotTo(HaveOccurred()) + Expect(result.Agents).To(HaveLen(2)) + }) + + // Cursor pagination: real multi-page listing with a NextPageToken + // round-trip, mirroring internal/policy/store/pagination_test.go. + It("returns a NextPageToken when more results exist, and paginates through all agents with it", func() { + names := []string{"page-agent-1", "page-agent-2", "page-agent-3", "page-agent-4", "page-agent-5"} + for _, n := range names { + _, err := agentStore.Create(ctx, newAgent(n)) + Expect(err).NotTo(HaveOccurred()) + } + + var seen []string + pageToken := "" + for { + result, err := agentStore.List(ctx, nil, &agent.Pagination{Limit: 2, PageToken: pageToken}) + Expect(err).NotTo(HaveOccurred()) + for _, a := range result.Agents { + seen = append(seen, a.Name) + } + if result.NextPageToken == "" { + break + } + pageToken = result.NextPageToken + } + + Expect(seen).To(Equal(names), "name ASC ordering must be stable across pages so the cursor never skips or repeats a row") + }) + + It("returns an empty NextPageToken on the last page", func() { + for i := 0; i < 2; i++ { + _, err := agentStore.Create(ctx, newAgent(uuid.New().String())) + Expect(err).NotTo(HaveOccurred()) + } + + result, err := agentStore.List(ctx, nil, &agent.Pagination{Limit: 2}) + + Expect(err).NotTo(HaveOccurred()) + Expect(result.Agents).To(HaveLen(2)) + Expect(result.NextPageToken).To(BeEmpty()) + }) + + It("returns ErrInvalidPageToken for a malformed page_token", func() { + _, err := agentStore.List(ctx, nil, &agent.Pagination{Limit: 2, PageToken: "not-valid-base64!!!"}) + + Expect(err).To(MatchError(agent.ErrInvalidPageToken)) + }) + }) + + Describe("Update", func() { + It("updates fields and returns updated agent", func() { + a := newAgent("to-update") + _, err := agentStore.Create(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + a.Environment = "staging" + updated, err := agentStore.Update(ctx, a) + + Expect(err).NotTo(HaveOccurred()) + Expect(updated.Environment).To(Equal("staging")) + }) + + It("returns ErrAgentNotFound for non-existing agent", func() { + a := newAgent("non-existing") + _, err := agentStore.Update(ctx, a) + + Expect(err).To(Equal(agent.ErrAgentNotFound)) + }) + + It("does not clobber a concurrently-applied newer heartbeat (R2 S6: finding #3)", func() { + // Simulates RegisterOrUpdate racing a genuine heartbeat: the + // heartbeat (UpdateHeartbeatIfNewer) lands first with a newer + // timestamp reporting "congested"; RegisterOrUpdate's own + // Update, built from a snapshot read before that heartbeat + // landed, must not unconditionally revert health_status back to + // "ready" with its own (older) timestamp. + a := newAgent("racing-agent") + a.HealthStatus = model.AgentHealthStatusReady + olderHB := time.Now() + a.LastHeartbeat = &olderHB + _, err := agentStore.Create(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + newerHB := olderHB.Add(time.Second) + applied, err := agentStore.UpdateHeartbeatIfNewer(ctx, a.ID, newerHB, model.AgentHealthStatusCongested) + Expect(err).NotTo(HaveOccurred()) + Expect(applied).To(BeTrue()) + + // RegisterOrUpdate's stale snapshot: still thinks last_heartbeat + // is olderHB and wants to force health_status back to "ready". + a.HealthStatus = model.AgentHealthStatusReady + a.LastHeartbeat = &olderHB + updated, err := agentStore.Update(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + Expect(updated.HealthStatus).To(Equal(model.AgentHealthStatusCongested)) + Expect(updated.LastHeartbeat.Unix()).To(Equal(newerHB.Unix())) + }) + }) + + Describe("Delete", func() { + It("removes the agent", func() { + a := newAgent("to-delete") + _, err := agentStore.Create(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + err = agentStore.Delete(ctx, a.ID) + + Expect(err).NotTo(HaveOccurred()) + + _, err = agentStore.Get(ctx, a.ID) + Expect(err).To(Equal(agent.ErrAgentNotFound)) + }) + + It("returns ErrAgentNotFound for missing ID", func() { + err := agentStore.Delete(ctx, uuid.New().String()) + + Expect(err).To(Equal(agent.ErrAgentNotFound)) + }) + }) + + Describe("UpdateHeartbeatIfNewer", func() { + It("applies when there is no prior heartbeat", func() { + a := newAgent("first-heartbeat") + _, err := agentStore.Create(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + applied, err := agentStore.UpdateHeartbeatIfNewer(ctx, a.ID, time.Now(), model.AgentHealthStatusReady) + Expect(err).NotTo(HaveOccurred()) + Expect(applied).To(BeTrue()) + }) + + It("applies a newer timestamp and rejects a stale one, closing the read-then-write race (L)", func() { + a := newAgent("monotonic-heartbeat") + _, err := agentStore.Create(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + newer := time.Now() + applied, err := agentStore.UpdateHeartbeatIfNewer(ctx, a.ID, newer, model.AgentHealthStatusReady) + Expect(err).NotTo(HaveOccurred()) + Expect(applied).To(BeTrue()) + + stale := newer.Add(-1 * time.Hour) + applied, err = agentStore.UpdateHeartbeatIfNewer(ctx, a.ID, stale, model.AgentHealthStatusCongested) + Expect(err).NotTo(HaveOccurred()) + Expect(applied).To(BeFalse()) + + found, err := agentStore.Get(ctx, a.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(found.LastHeartbeat).NotTo(BeNil()) + Expect(*found.LastHeartbeat).To(BeTemporally("~", newer, time.Millisecond)) + // The stale write's health_status must not have landed either - + // the whole update (both columns) is guarded by the same CAS. + Expect(found.HealthStatus).To(Equal(model.AgentHealthStatusReady)) + }) + + It("returns applied=false for an unknown agent", func() { + applied, err := agentStore.UpdateHeartbeatIfNewer(ctx, uuid.New().String(), time.Now(), model.AgentHealthStatusReady) + Expect(err).NotTo(HaveOccurred()) + Expect(applied).To(BeFalse()) + }) + }) + + Describe("MarkStaleUnavailable", func() { + It("flips agents whose heartbeat is older than cutoff to Unavailable in one pass", func() { + stale := newAgent("stale-agent") + _, err := agentStore.Create(ctx, stale) + Expect(err).NotTo(HaveOccurred()) + oldHB := time.Now().Add(-1 * time.Hour) + _, err = agentStore.UpdateHeartbeatIfNewer(ctx, stale.ID, oldHB, model.AgentHealthStatusReady) + Expect(err).NotTo(HaveOccurred()) + + fresh := newAgent("fresh-agent") + _, err = agentStore.Create(ctx, fresh) + Expect(err).NotTo(HaveOccurred()) + _, err = agentStore.UpdateHeartbeatIfNewer(ctx, fresh.ID, time.Now(), model.AgentHealthStatusReady) + Expect(err).NotTo(HaveOccurred()) + + cutoff := time.Now().Add(-30 * time.Minute) + Expect(agentStore.MarkStaleUnavailable(ctx, cutoff)).To(Succeed()) + + staleFound, err := agentStore.Get(ctx, stale.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(staleFound.HealthStatus).To(Equal(model.AgentHealthStatusUnavailable)) + + freshFound, err := agentStore.Get(ctx, fresh.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(freshFound.HealthStatus).To(Equal(model.AgentHealthStatusReady)) + }) + + It("does not clobber a heartbeat that lands concurrently with the sweep (M)", func() { + // Simulates the race the read-then-write version of sweep() was + // vulnerable to: MarkStaleUnavailable re-checks staleness in the + // same statement as the write, so a heartbeat recorded AFTER + // cutoff was computed (but before/at the same moment the sweep + // runs) is never overwritten back to Unavailable. + a := newAgent("race-agent") + _, err := agentStore.Create(ctx, a) + Expect(err).NotTo(HaveOccurred()) + + cutoff := time.Now() + _, err = agentStore.UpdateHeartbeatIfNewer(ctx, a.ID, time.Now(), model.AgentHealthStatusReady) + Expect(err).NotTo(HaveOccurred()) + + Expect(agentStore.MarkStaleUnavailable(ctx, cutoff)).To(Succeed()) + + found, err := agentStore.Get(ctx, a.ID) + Expect(err).NotTo(HaveOccurred()) + Expect(found.HealthStatus).To(Equal(model.AgentHealthStatusReady)) + }) + }) + + Describe("ListReady", func() { + It("returns only agents with health_status ready", func() { + a1 := newAgent("ready-1") + _, err := agentStore.Create(ctx, a1) + Expect(err).NotTo(HaveOccurred()) + + a2 := newAgent("congested-1") + a2.HealthStatus = model.AgentHealthStatusCongested + _, err = agentStore.Create(ctx, a2) + Expect(err).NotTo(HaveOccurred()) + + a3 := newAgent("ready-2") + _, err = agentStore.Create(ctx, a3) + Expect(err).NotTo(HaveOccurred()) + + agents, err := agentStore.ListReady(ctx) + + Expect(err).NotTo(HaveOccurred()) + Expect(agents).To(HaveLen(2)) + }) + }) +}) + +func newAgent(name string) model.Agent { + return model.Agent{ + ID: uuid.New().String(), + Name: name, + Environment: "production", + ServiceTypes: []string{"vm", "container"}, + TopicName: "dcm.agent." + name, + HealthStatus: model.AgentHealthStatusReady, + } +} diff --git a/internal/agent/store/agent/pagination.go b/internal/agent/store/agent/pagination.go new file mode 100644 index 0000000..2ede7b9 --- /dev/null +++ b/internal/agent/store/agent/pagination.go @@ -0,0 +1,42 @@ +package agent + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" +) + +// ErrInvalidPageToken is returned when page_token cannot be decoded or validated. +var ErrInvalidPageToken = errors.New("store: invalid page_token") + +type pageTokenPayload struct { + Offset int `json:"offset"` +} + +func decodePageToken(token string) (int, error) { + decoded, err := base64.StdEncoding.DecodeString(token) + if err != nil { + return 0, fmt.Errorf("%w: %v", ErrInvalidPageToken, err) + } + + var payload pageTokenPayload + if err := json.Unmarshal(decoded, &payload); err != nil { + return 0, fmt.Errorf("%w: %v", ErrInvalidPageToken, err) + } + if payload.Offset < 0 { + return 0, fmt.Errorf("%w: negative offset", ErrInvalidPageToken) + } + return payload.Offset, nil +} + +func encodePageToken(offset int) (string, error) { + if offset < 0 { + return "", fmt.Errorf("%w: negative offset", ErrInvalidPageToken) + } + payload, err := json.Marshal(pageTokenPayload{Offset: offset}) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(payload), nil +} diff --git a/internal/agent/store/model/agent.go b/internal/agent/store/model/agent.go new file mode 100644 index 0000000..cb47c1a --- /dev/null +++ b/internal/agent/store/model/agent.go @@ -0,0 +1,42 @@ +// Package model defines the Agent domain model for GORM persistence. +package model + +import ( + "time" +) + +type AgentHealthStatus string + +const ( + AgentHealthStatusReady AgentHealthStatus = "ready" + AgentHealthStatusCongested AgentHealthStatus = "congested" + AgentHealthStatusUnavailable AgentHealthStatus = "unavailable" +) + +// AgentCost is the relative cost weight an agent reports for placement +// decisions. It mirrors the enum enforced by the OpenAPI request-validation +// middleware in front of /agents; Go code does not re-validate membership. +type AgentCost string + +const ( + AgentCostLow AgentCost = "low" + AgentCostMediumLow AgentCost = "medium-low" + AgentCostMedium AgentCost = "medium" + AgentCostMediumHigh AgentCost = "medium-high" + AgentCostHigh AgentCost = "high" +) + +type Agent struct { + ID string `gorm:"primaryKey;type:varchar(63)"` + Name string `gorm:"uniqueIndex;not null"` + Environment string `gorm:"column:environment"` + ServiceTypes []string `gorm:"column:service_types;serializer:json"` + Cost *AgentCost `gorm:"column:cost;type:varchar(16)"` + TopicName string `gorm:"column:topic_name;not null;uniqueIndex"` + HealthStatus AgentHealthStatus `gorm:"column:health_status;default:ready;index:idx_health_heartbeat"` + LastHeartbeat *time.Time `gorm:"column:last_heartbeat;index:idx_health_heartbeat"` + CreateTime time.Time `gorm:"column:create_time;autoCreateTime"` + UpdateTime time.Time `gorm:"column:update_time;autoUpdateTime"` +} + +type AgentList []Agent diff --git a/internal/app/config.go b/internal/app/config.go index 178622d..7a9b4f6 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -19,9 +19,21 @@ type Config struct { Seed SeedConfig NATS NATSConfig SP SPConfig + Agent AgentConfig Wiring WiringConfig } +type AgentConfig struct { + HeartbeatTimeout time.Duration `envconfig:"AGENT_HEARTBEAT_TIMEOUT" default:"60s"` + ConsumerLagThreshold int64 `envconfig:"AGENT_CONSUMER_LAG_THRESHOLD" default:"100"` + QueuedRequestTimeout time.Duration `envconfig:"AGENT_QUEUED_REQUEST_TIMEOUT" default:"5m"` + PendingRequestTimeout time.Duration `envconfig:"AGENT_PENDING_REQUEST_TIMEOUT" default:"2m"` + PendingRequestMaxRetries int `envconfig:"AGENT_PENDING_REQUEST_MAX_RETRIES" default:"3"` + SweepInterval time.Duration `envconfig:"AGENT_SWEEP_INTERVAL" default:"10s"` + ResponseMaxDeliver int `envconfig:"AGENT_RESPONSE_MAX_DELIVER" default:"10"` + ResponseAckWait time.Duration `envconfig:"AGENT_RESPONSE_ACK_WAIT" default:"30s"` +} + type AuthConfig struct { Disabled bool `envconfig:"AUTH_DISABLED" default:"true"` ProxySecret string `envconfig:"AUTH_PROXY_SECRET"` diff --git a/internal/app/db.go b/internal/app/db.go index aed57dc..bbe110e 100644 --- a/internal/app/db.go +++ b/internal/app/db.go @@ -6,6 +6,7 @@ import ( "strings" "time" + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" authmodel "github.com/dcm-project/control-plane/internal/auth/store/model" catalogmodel "github.com/dcm-project/control-plane/internal/catalog/store/model" placementmodel "github.com/dcm-project/control-plane/internal/placement/store/model" @@ -69,6 +70,7 @@ func openDB(cfg *Config) (*gorm.DB, error) { sqlDB.SetConnMaxLifetime(time.Hour) if err := db.AutoMigrate( + &agentmodel.Agent{}, &authmodel.Actor{}, &authmodel.ActorIdentity{}, &catalogmodel.ServiceType{}, @@ -76,7 +78,6 @@ func openDB(cfg *Config) (*gorm.DB, error) { &catalogmodel.CatalogItemInstance{}, &placementmodel.Resource{}, &policymodel.Policy{}, - &spmodel.Provider{}, &spmodel.ServiceTypeInstance{}, ); err != nil { return nil, fmt.Errorf("migrate schema: %w", err) diff --git a/internal/app/openapi.go b/internal/app/openapi.go index 97183cc..7fb7870 100644 --- a/internal/app/openapi.go +++ b/internal/app/openapi.go @@ -10,9 +10,9 @@ import ( "github.com/dcm-project/control-plane/internal/auth" + agentapi "github.com/dcm-project/control-plane/api/agent/v1alpha1" catalogapi "github.com/dcm-project/control-plane/api/catalog/v1alpha1" policyapi "github.com/dcm-project/control-plane/api/policy/v1alpha1" - spproviderapi "github.com/dcm-project/control-plane/api/sp/v1alpha1/provider" sprmapi "github.com/dcm-project/control-plane/api/sp/v1alpha1/resource_manager" "github.com/getkin/kin-openapi/openapi3" "github.com/getkin/kin-openapi/openapi3filter" @@ -22,13 +22,18 @@ import ( const apiV1Alpha1Prefix = "/api/v1alpha1" type openAPIValidators struct { - catalog func(http.Handler) http.Handler - policy func(http.Handler) http.Handler - provider func(http.Handler) http.Handler - rm func(http.Handler) http.Handler + agent func(http.Handler) http.Handler + catalog func(http.Handler) http.Handler + policy func(http.Handler) http.Handler + rm func(http.Handler) http.Handler } func newOpenAPIValidators() (*openAPIValidators, error) { + agentSpec, err := agentapi.GetSpec() + if err != nil { + return nil, fmt.Errorf("load agent OpenAPI spec: %w", err) + } + catalogSpec, err := catalogapi.GetSpec() if err != nil { return nil, fmt.Errorf("load catalog OpenAPI spec: %w", err) @@ -39,31 +44,23 @@ func newOpenAPIValidators() (*openAPIValidators, error) { return nil, fmt.Errorf("load policy OpenAPI spec: %w", err) } - providerSpec, err := spproviderapi.GetSpec() - if err != nil { - return nil, fmt.Errorf("load service provider OpenAPI spec: %w", err) - } - rmSpec, err := sprmapi.GetSpec() if err != nil { return nil, fmt.Errorf("load resource manager OpenAPI spec: %w", err) } return &openAPIValidators{ - catalog: oapiRequestValidator(catalogSpec), - policy: oapiRequestValidator(policySpec), - provider: oapiRequestValidator(providerSpec), - rm: oapiRequestValidator(rmSpec), + agent: oapiRequestValidator(agentSpec), + catalog: oapiRequestValidator(catalogSpec), + policy: oapiRequestValidator(policySpec), + rm: oapiRequestValidator(rmSpec), }, nil } func oapiRequestValidator(spec *openapi3.T) func(http.Handler) http.Handler { return nethttpmiddleware.OapiRequestValidatorWithOptions(spec, &nethttpmiddleware.Options{ Options: openapi3filter.Options{ - AuthenticationFunc: verifyActorContext, - // kin-openapi rewrites validated bodies when schema defaults are applied, - // but only registers encoders for application/json. PATCH merge bodies - // use application/merge-patch+json and must stay partial (RFC 7396). + AuthenticationFunc: verifyActorContext, SkipSettingDefaults: true, }, SilenceServersWarning: true, @@ -116,10 +113,10 @@ func (v *openAPIValidators) middleware() func(http.Handler) http.Handler { } switch { + case strings.HasPrefix(path, apiV1Alpha1Prefix+"/agents"): + v.agent(next).ServeHTTP(w, r) case strings.HasPrefix(path, apiV1Alpha1Prefix+"/service-type-instances"): v.rm(next).ServeHTTP(w, r) - case strings.HasPrefix(path, apiV1Alpha1Prefix+"/providers"): - v.provider(next).ServeHTTP(w, r) case strings.HasPrefix(path, apiV1Alpha1Prefix+"/policies"): v.policy(next).ServeHTTP(w, r) default: diff --git a/internal/app/openapi_validation_test.go b/internal/app/openapi_validation_test.go index 2507daf..63c9b6d 100644 --- a/internal/app/openapi_validation_test.go +++ b/internal/app/openapi_validation_test.go @@ -66,12 +66,6 @@ var _ = Describe("OpenAPI request validation", func() { }) }) - Describe("SP provider routes", func() { - It("rejects malformed JSON on POST /providers", func() { - expectInvalidJSONRejected(validators, "/api/v1alpha1/providers") - }) - }) - Describe("SP resource manager routes", func() { It("rejects malformed JSON on POST /service-type-instances", func() { expectInvalidJSONRejected(validators, "/api/v1alpha1/service-type-instances") diff --git a/internal/app/run.go b/internal/app/run.go index b1dff8a..098c86e 100644 --- a/internal/app/run.go +++ b/internal/app/run.go @@ -12,6 +12,11 @@ import ( "syscall" "time" + agentserver "github.com/dcm-project/control-plane/internal/agent/api/server" + agenthandlers "github.com/dcm-project/control-plane/internal/agent/handlers/v1alpha1" + agenthealthcheck "github.com/dcm-project/control-plane/internal/agent/healthcheck" + agentservice "github.com/dcm-project/control-plane/internal/agent/service" + agentstore "github.com/dcm-project/control-plane/internal/agent/store/agent" "github.com/dcm-project/control-plane/internal/auth" authservice "github.com/dcm-project/control-plane/internal/auth/service" authstore "github.com/dcm-project/control-plane/internal/auth/store" @@ -21,6 +26,7 @@ import ( catalogplacement "github.com/dcm-project/control-plane/internal/catalog/placement" catalogservice "github.com/dcm-project/control-plane/internal/catalog/service" catalogstore "github.com/dcm-project/control-plane/internal/catalog/store" + placementagent "github.com/dcm-project/control-plane/internal/placement/agent" placementlogging "github.com/dcm-project/control-plane/internal/placement/logging" placementpolicy "github.com/dcm-project/control-plane/internal/placement/policy" placementservice "github.com/dcm-project/control-plane/internal/placement/service" @@ -32,25 +38,24 @@ import ( policyopa "github.com/dcm-project/control-plane/internal/policy/opa" policyservice "github.com/dcm-project/control-plane/internal/policy/service" policystore "github.com/dcm-project/control-plane/internal/policy/store" - spproviderserver "github.com/dcm-project/control-plane/internal/sp/api/provider" sprmserver "github.com/dcm-project/control-plane/internal/sp/api/resource_manager" spcleanup "github.com/dcm-project/control-plane/internal/sp/cleanup" spconfig "github.com/dcm-project/control-plane/internal/sp/config" spconsumer "github.com/dcm-project/control-plane/internal/sp/consumer" - spproviderhandler "github.com/dcm-project/control-plane/internal/sp/handlers/provider" sprmhandler "github.com/dcm-project/control-plane/internal/sp/handlers/resource_manager" - sphealthcheck "github.com/dcm-project/control-plane/internal/sp/healthcheck" splogging "github.com/dcm-project/control-plane/internal/sp/logging" - spprovidersvc "github.com/dcm-project/control-plane/internal/sp/service/provider" + "github.com/dcm-project/control-plane/internal/sp/messaging" + sppending "github.com/dcm-project/control-plane/internal/sp/pending" sprmsvc "github.com/dcm-project/control-plane/internal/sp/service/resource_manager" spstore "github.com/dcm-project/control-plane/internal/sp/store" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" ) const gracefulShutdownTimeout = 5 * time.Second -// Run starts the control-plane monolith. func Run() int { cfg, err := LoadConfig() if err != nil { @@ -87,6 +92,10 @@ func Run() int { placementDataStore := placementstore.NewStore(db) spDataStore := spstore.NewStore(db) + agentSt := agentstore.NewAgent(db) + agentSvc := agentservice.NewAgentService(agentSt, cfg.Agent.ConsumerLagThreshold) + agentClient := placementagent.NewServiceClient(agentSt) + opaEngine := policyopa.NewEngine() policyService := policyservice.NewPolicyService(policyDataStore, opaEngine) evaluationService := policyservice.NewEvaluationService(policyDataStore.Policy(), opaEngine) @@ -95,13 +104,64 @@ func Run() int { return 1 } - spProviderService := spprovidersvc.NewProviderService(spDataStore) - spInstanceService := sprmsvc.NewInstanceService(spDataStore, nil) + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + var publisher *messaging.Publisher + checkers := []Checker{NewPostgresChecker(db)} + + if !cfg.NATS.Disabled { + statusConsumer, err := spconsumer.New(cfg.NATS.URL, cfg.NATS.Subject, spDataStore, + spconsumer.SetStreamName(cfg.NATS.StreamName), + spconsumer.SetConsumerName(cfg.NATS.ConsumerName), + ) + if err != nil { + slog.Error("Failed to initialize status consumer", "error", err) + return 1 + } + if err := statusConsumer.Start(ctx); err != nil { + slog.Error("Failed to start status consumer", "error", err) + return 1 + } + defer statusConsumer.Stop() + checkers = append(checkers, NewNATSChecker(statusConsumer)) + + agentNc, err := nats.Connect(cfg.NATS.URL, nats.MaxReconnects(-1)) + if err != nil { + slog.Error("Failed to connect to NATS for agent response consumer", "error", err) + return 1 + } + defer agentNc.Close() + agentJS, err := jetstream.New(agentNc) + if err != nil { + slog.Error("Failed to create JetStream for agent response consumer", "error", err) + return 1 + } + + publisher = messaging.NewPublisher(agentJS) + + if err := publisher.EnsureStream(ctx); err != nil { + slog.Error("Failed to ensure agent request stream", "error", err) + return 1 + } + + responseConsumer := spconsumer.NewResponseConsumer(agentJS, spDataStore, agentSt, cfg.Agent.ResponseMaxDeliver, cfg.Agent.ResponseAckWait) + if err := responseConsumer.Start(ctx); err != nil { + slog.Error("Failed to start agent response consumer", "error", err) + return 1 + } + defer responseConsumer.Stop() + } + + spInstanceService := sprmsvc.NewInstanceService(spDataStore, publisher, agentSt) policyClient := placementpolicy.NewServiceClient(evaluationService) sprmClient := placementsprm.NewServiceClient(spInstanceService) - placementService := placementservice.NewPlacementService(placementDataStore, policyClient, sprmClient) + placementService := placementservice.NewPlacementService( + placementDataStore, policyClient, sprmClient, + placementservice.WithAgentClient(agentClient), + ) pmClient, err := buildPlacementClient(cfg, placementService, logger) if err != nil { slog.Error("Failed to initialize placement client", "error", err) @@ -118,9 +178,6 @@ func Run() int { return 1 } - ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer cancel() - if err := authSvc.Seed(ctx); err != nil { slog.Error("Failed to seed auth database", "error", err) return 1 @@ -134,40 +191,15 @@ func Run() int { return 1 } - checkers := []Checker{NewPostgresChecker(db)} - - if !cfg.NATS.Disabled { - statusConsumer, err := spconsumer.New(cfg.NATS.URL, cfg.NATS.Subject, spDataStore, - spconsumer.SetStreamName(cfg.NATS.StreamName), - spconsumer.SetConsumerName(cfg.NATS.ConsumerName), - ) - if err != nil { - slog.Error("Failed to initialize status consumer", "error", err) - return 1 - } - if err := statusConsumer.Start(ctx); err != nil { - slog.Error("Failed to start status consumer", "error", err) - return 1 - } - defer statusConsumer.Stop() - checkers = append(checkers, NewNATSChecker(statusConsumer)) - } + agentSweep := sppending.NewSweep(db, publisher, agentSt, placementService, cfg.Agent.PendingRequestTimeout, cfg.Agent.QueuedRequestTimeout, cfg.Agent.SweepInterval, cfg.Agent.PendingRequestMaxRetries) + agentSweep.Start(ctx) + defer agentSweep.Stop() - healthMonitor := sphealthcheck.NewMonitor( - spDataStore.Provider(), - spDataStore.ServiceTypeInstance(), - &spconfig.HealthCheckConfig{ - Interval: cfg.SP.HealthCheckInterval, - Timeout: cfg.SP.HealthCheckTimeout, - MaxConsecutiveFailures: cfg.SP.HealthCheckMaxConsecutiveFailures, - BaseBackoffInterval: cfg.SP.HealthCheckBaseBackoffInterval, - MaxBackoffInterval: cfg.SP.HealthCheckMaxBackoffInterval, - }, - ) - healthMonitor.Start(ctx) - defer healthMonitor.Stop() + agentHealthMonitor := agenthealthcheck.NewMonitor(agentSt, cfg.Agent.HeartbeatTimeout, cfg.SP.HealthCheckInterval) + agentHealthMonitor.Start(ctx) + defer agentHealthMonitor.Stop() - cleanupScheduler := spcleanup.NewScheduler(spDataStore, spInstanceService, &spconfig.CleanupConfig{ + cleanupScheduler := spcleanup.NewScheduler(spDataStore, publisher, agentSt, &spconfig.CleanupConfig{ Interval: cfg.SP.CleanupInterval, MaxRetries: cfg.SP.CleanupMaxRetries, Timeout: cfg.SP.CleanupTimeout, @@ -206,10 +238,10 @@ func Run() int { } router, err := newRouter(authMiddleware, RouteHandlers{ - Catalog: cataloghandlers.NewHandler(catalogSvc, logger), - Policy: policyhandlers.NewPolicyHandler(policyService), - SPProvider: spproviderhandler.NewHandler(spProviderService), - SPRM: sprmhandler.NewHandler(spInstanceService), + Agent: agenthandlers.NewHandler(agentSvc), + Catalog: cataloghandlers.NewHandler(catalogSvc, logger), + Policy: policyhandlers.NewPolicyHandler(policyService), + SPRM: sprmhandler.NewHandler(spInstanceService), }, checkers...) if err != nil { slog.Error("Failed to configure HTTP router", "error", err) @@ -249,10 +281,10 @@ func Run() int { } type RouteHandlers struct { - Catalog catalogserver.StrictServerInterface - Policy policyserver.StrictServerInterface - SPProvider spproviderserver.StrictServerInterface - SPRM sprmserver.StrictServerInterface + Agent agentserver.StrictServerInterface + Catalog catalogserver.StrictServerInterface + Policy policyserver.StrictServerInterface + SPRM sprmserver.StrictServerInterface } func newRouter(authMW func(http.Handler) http.Handler, h RouteHandlers, checkers ...Checker) (chi.Router, error) { @@ -269,10 +301,13 @@ func newRouter(authMW func(http.Handler) http.Handler, h RouteHandlers, checkers const baseURL = "/api/v1alpha1" - // Single monolith health endpoint; domain OpenAPI specs omit /health to avoid - // duplicate chi route registration when mounting multiple generated servers. registerMonolithHealth(router, checkers...) + agentserver.HandlerFromMuxWithBaseURL( + agentserver.NewStrictHandler(h.Agent, nil), + router, + baseURL, + ) catalogserver.HandlerFromMuxWithBaseURL( catalogserver.NewStrictHandler(h.Catalog, nil), router, @@ -285,10 +320,6 @@ func newRouter(authMW func(http.Handler) http.Handler, h RouteHandlers, checkers ) apiRouter := chi.NewRouter() - spproviderserver.HandlerFromMux( - spproviderserver.NewStrictHandler(h.SPProvider, nil), - apiRouter, - ) sprmserver.HandlerFromMux( sprmserver.NewStrictHandler(h.SPRM, nil), apiRouter, diff --git a/internal/catalog/placement/local_client.go b/internal/catalog/placement/local_client.go index cf7c26c..315fbcb 100644 --- a/internal/catalog/placement/local_client.go +++ b/internal/catalog/placement/local_client.go @@ -94,13 +94,27 @@ func mapPlacementServiceError(err error) error { status = http.StatusNotAcceptable case service.ErrCodeConflict, service.ErrCodePolicyConflict: status = http.StatusConflict - case service.ErrCodeProviderError: + case service.ErrCodeProvisioningError: status = http.StatusUnprocessableEntity case service.ErrCodeNotFound: status = http.StatusNotFound case service.ErrCodePolicyError, service.ErrCodePolicyInternalError: status = http.StatusFailedDependency + case service.ErrCodeUnavailable: + status = http.StatusServiceUnavailable } - return &PlacementError{StatusCode: status, Body: svcErr.Message} + // 5xx bodies may carry internal error detail from downstream services - + // log server-side and return a generic message to the API caller. + body := svcErr.Message + if status >= http.StatusInternalServerError { + slog.Error("placement local client error", "status", status, "detail", svcErr.Message) + if status == http.StatusServiceUnavailable { + body = "service temporarily unavailable" + } else { + body = "internal server error" + } + } + + return &PlacementError{StatusCode: status, Body: body} } diff --git a/internal/catalog/service/catalog_item_instance.go b/internal/catalog/service/catalog_item_instance.go index fa765aa..c7bc764 100644 --- a/internal/catalog/service/catalog_item_instance.go +++ b/internal/catalog/service/catalog_item_instance.go @@ -271,7 +271,10 @@ func (s *catalogItemInstanceService) Delete(ctx context.Context, id string) erro s.logger.DebugContext(ctx, "Calling placement manager to delete run", "id", id, "run_id", instance.RunID) if err := s.pmClient.DeleteRun(ctx, instance.RunID); err != nil { s.logger.ErrorContext(ctx, "Placement manager delete failed", "id", id, "error", err) - return fmt.Errorf("%w: %s", ErrPlacementManagerDeleteFailed, err.Error()) + // mapPlacementError, not a direct wrap: distinguishes + // policy-rejected/provider-error/policy-dependency (406/422/424) + // from a generic placement failure, matching create/rehydrate. + return mapPlacementError(err, ErrPlacementManagerDeleteFailed) } err = s.store.CatalogItemInstance().Delete(ctx, id) diff --git a/internal/catalog/service/catalog_item_instance_test.go b/internal/catalog/service/catalog_item_instance_test.go index c340f86..1fe4a87 100644 --- a/internal/catalog/service/catalog_item_instance_test.go +++ b/internal/catalog/service/catalog_item_instance_test.go @@ -1016,5 +1016,50 @@ var _ = Describe("CatalogItemInstance Service with Placement Manager", func() { Expect(getErr).ToNot(HaveOccurred()) Expect(result).ToNot(BeNil()) }) + + // Delete must distinguish PM failure kinds via mapPlacementError, + // like create/rehydrate, instead of a generic sentinel. + It("should return ErrPlacementManagerPolicyRejected when PM delete returns 406", func() { + instanceID := "delete-policy-fail" + seedCatalogItemInstance(ctx, str, instanceID) + + mockPM.deleteFunc = func(_ context.Context, _ string) error { + return &placement.PlacementError{StatusCode: 406, Body: "policy rejected"} + } + + err := svc.CatalogItemInstance().Delete(ctx, instanceID) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, service.ErrPlacementManagerPolicyRejected)).To(BeTrue()) + + result, getErr := svc.CatalogItemInstance().Get(ctx, instanceID) + Expect(getErr).ToNot(HaveOccurred()) + Expect(result).ToNot(BeNil()) + }) + + It("should return ErrPlacementManagerProviderError when PM delete returns 422", func() { + instanceID := "delete-provider-fail" + seedCatalogItemInstance(ctx, str, instanceID) + + mockPM.deleteFunc = func(_ context.Context, _ string) error { + return &placement.PlacementError{StatusCode: 422, Body: "provider error"} + } + + err := svc.CatalogItemInstance().Delete(ctx, instanceID) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, service.ErrPlacementManagerProviderError)).To(BeTrue()) + }) + + It("should return ErrPlacementManagerPolicyDependency when PM delete returns 424", func() { + instanceID := "delete-dependency-fail" + seedCatalogItemInstance(ctx, str, instanceID) + + mockPM.deleteFunc = func(_ context.Context, _ string) error { + return &placement.PlacementError{StatusCode: 424, Body: "policy dependency"} + } + + err := svc.CatalogItemInstance().Delete(ctx, instanceID) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, service.ErrPlacementManagerPolicyDependency)).To(BeTrue()) + }) }) }) diff --git a/internal/placement/agent/service_client.go b/internal/placement/agent/service_client.go new file mode 100644 index 0000000..de79bd6 --- /dev/null +++ b/internal/placement/agent/service_client.go @@ -0,0 +1,39 @@ +package agent + +import ( + "context" + + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" +) + +// readyLister is the single method this adapter needs from agentstore.Agent. +// Depending on this narrow interface, rather than the full store interface, +// keeps the compile-time surface honest about what's actually used here. +type readyLister interface { + ListReady(ctx context.Context) (agentmodel.AgentList, error) +} + +type serviceClient struct { + store readyLister +} + +// NewServiceClient adapts the agent store for in-process use by PlacementService. +func NewServiceClient(store readyLister) Client { + return &serviceClient{store: store} +} + +func (c *serviceClient) ListReadyAgents(ctx context.Context) ([]Info, error) { + agents, err := c.store.ListReady(ctx) + if err != nil { + return nil, err + } + infos := make([]Info, len(agents)) + for i, a := range agents { + cost := "" + if a.Cost != nil { + cost = string(*a.Cost) + } + infos[i] = Info{Name: a.Name, Environment: a.Environment, ServiceTypes: a.ServiceTypes, Cost: cost} + } + return infos, nil +} diff --git a/internal/placement/agent/service_client_test.go b/internal/placement/agent/service_client_test.go new file mode 100644 index 0000000..61fa878 --- /dev/null +++ b/internal/placement/agent/service_client_test.go @@ -0,0 +1,63 @@ +package agent + +import ( + "context" + "testing" + + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" +) + +type fakeReadyLister struct { + agents agentmodel.AgentList + err error +} + +func (f *fakeReadyLister) ListReady(_ context.Context) (agentmodel.AgentList, error) { + return f.agents, f.err +} + +// TestListReadyAgents_ThreadsAllFields guards against the exact bug this +// change fixed: an Agent field silently dropped on the way into the +// placement/agent.Info the policy engine ultimately sees. +func TestListReadyAgents_ThreadsAllFields(t *testing.T) { + costLow := agentmodel.AgentCostLow + lister := &fakeReadyLister{agents: agentmodel.AgentList{ + {Name: "agent-a", Environment: "prod", ServiceTypes: []string{"vm", "database"}, Cost: &costLow}, + {Name: "agent-b", Environment: "dev", ServiceTypes: nil, Cost: nil}, + }} + client := NewServiceClient(lister) + + infos, err := client.ListReadyAgents(context.Background()) + if err != nil { + t.Fatalf("ListReadyAgents returned unexpected error: %v", err) + } + if len(infos) != 2 { + t.Fatalf("got %d agents, want 2", len(infos)) + } + + a := infos[0] + if a.Name != "agent-a" || a.Environment != "prod" || a.Cost != "low" { + t.Errorf("agent-a = %+v, want Name=agent-a Environment=prod Cost=low", a) + } + if len(a.ServiceTypes) != 2 || a.ServiceTypes[0] != "vm" || a.ServiceTypes[1] != "database" { + t.Errorf("agent-a ServiceTypes = %v, want [vm database]", a.ServiceTypes) + } + + b := infos[1] + if b.Cost != "" { + t.Errorf("agent-b Cost = %q, want \"\" (nil *AgentCost dereferenced safely)", b.Cost) + } + if b.ServiceTypes != nil { + t.Errorf("agent-b ServiceTypes = %v, want nil (passed through unchanged)", b.ServiceTypes) + } +} + +func TestListReadyAgents_PropagatesStoreError(t *testing.T) { + lister := &fakeReadyLister{err: context.DeadlineExceeded} + client := NewServiceClient(lister) + + _, err := client.ListReadyAgents(context.Background()) + if err == nil { + t.Fatal("expected an error, got nil") + } +} diff --git a/internal/placement/agent/types.go b/internal/placement/agent/types.go new file mode 100644 index 0000000..47e5470 --- /dev/null +++ b/internal/placement/agent/types.go @@ -0,0 +1,20 @@ +// Package agent provides types and an adapter boundary for listing ready +// agents, mirroring the policy and sprm adapter packages. +package agent + +import "context" + +// Info is the subset of agent metadata exposed to PlacementService for +// policy evaluation. Cost is "" when the agent didn't report one (it's an +// optional field at registration, unlike Name/ServiceTypes). +type Info struct { + Name string + Environment string + ServiceTypes []string + Cost string +} + +// Client is the port PlacementService uses to list ready agents. +type Client interface { + ListReadyAgents(ctx context.Context) ([]Info, error) +} diff --git a/internal/placement/policy/service_client.go b/internal/placement/policy/service_client.go index 0df7106..4ca4884 100644 --- a/internal/placement/policy/service_client.go +++ b/internal/placement/policy/service_client.go @@ -19,8 +19,15 @@ func NewServiceClient(eval policyservice.EvaluationService) Client { } func (c *serviceClient) Evaluate(ctx context.Context, req EvaluateRequest) (*EvaluateResponse, error) { + availableAgents := make([]policyservice.AgentInfo, len(req.AvailableAgents)) + for i, a := range req.AvailableAgents { + availableAgents[i] = policyservice.AgentInfo{Name: a.Name, Environment: a.Environment, ServiceTypes: a.ServiceTypes, Cost: a.Cost} + } + response, err := c.eval.EvaluateRequest(ctx, &policyservice.EvaluationRequest{ ServiceInstance: req.Spec, + AvailableAgents: availableAgents, + ExcludeAgents: req.ExcludeAgents, }) if err != nil { return nil, mapEvaluationError(err) @@ -28,9 +35,9 @@ func (c *serviceClient) Evaluate(ctx context.Context, req EvaluateRequest) (*Eva status := string(response.Status) return &EvaluateResponse{ - Status: status, - SelectedProvider: response.SelectedProvider, - EvaluatedSpec: response.EvaluatedServiceInstance, + Status: status, + SelectedAgent: response.SelectedAgent, + EvaluatedSpec: response.EvaluatedServiceInstance, }, nil } diff --git a/internal/placement/policy/service_client_test.go b/internal/placement/policy/service_client_test.go new file mode 100644 index 0000000..42f18d6 --- /dev/null +++ b/internal/placement/policy/service_client_test.go @@ -0,0 +1,155 @@ +package policy + +import ( + "context" + "errors" + "net/http" + "testing" + + policyservice "github.com/dcm-project/control-plane/internal/policy/service" +) + +// TestMapEvaluationError checks the ServiceError.Type-to-status mapping, +// including the new NewNoCapableAgentError/NewAllAgentsExcludedError paths +// (both ErrorTypeRejected), and that a non-ServiceError is wrapped rather +// than silently mapped to a default status. +func TestMapEvaluationError(t *testing.T) { + cases := []struct { + name string + err error + wantStatusCode int // 0 means "expect no *HTTPError at all" + }{ + { + name: "invalid argument maps to 400", + err: policyservice.NewInvalidArgumentError("bad spec", "detail"), + wantStatusCode: http.StatusBadRequest, + }, + { + name: "policy rejected maps to 406", + err: policyservice.NewPolicyRejectedError("policy-1", "denied"), + wantStatusCode: http.StatusNotAcceptable, + }, + { + name: "no capable agent maps to 406 (same bucket as policy rejection)", + err: policyservice.NewNoCapableAgentError("database", 1), + wantStatusCode: http.StatusNotAcceptable, + }, + { + name: "all agents excluded maps to 406 (same bucket as policy rejection)", + err: policyservice.NewAllAgentsExcludedError(2), + wantStatusCode: http.StatusNotAcceptable, + }, + { + name: "policy conflict maps to 409", + err: policyservice.NewPolicyConflictError("low-prio", "field", "high-prio"), + wantStatusCode: http.StatusConflict, + }, + { + name: "internal error maps to the 500 default", + err: policyservice.NewInternalError("failed", "detail", nil), + wantStatusCode: http.StatusInternalServerError, + }, + { + name: "a non-ServiceError is wrapped rather than mapped to an HTTPError", + err: errors.New("some unrelated infra error"), + wantStatusCode: 0, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := mapEvaluationError(tc.err) + + var httpErr *HTTPError + ok := errors.As(got, &httpErr) + + if tc.wantStatusCode == 0 { + if ok { + t.Fatalf("expected a non-HTTPError wrapped error, got *HTTPError{%d, %q}", httpErr.StatusCode, httpErr.Body) + } + return + } + + if !ok { + t.Fatalf("expected *HTTPError, got %T: %v", got, got) + } + if httpErr.StatusCode != tc.wantStatusCode { + t.Errorf("StatusCode = %d, want %d", httpErr.StatusCode, tc.wantStatusCode) + } + }) + } +} + +// capturingEvaluationService records the EvaluationRequest it was called +// with, so tests can assert on the AgentInfo conversion Evaluate performs. +type capturingEvaluationService struct { + captured *policyservice.EvaluationRequest + response *policyservice.EvaluationResponse + err error +} + +func (c *capturingEvaluationService) EvaluateRequest(_ context.Context, req *policyservice.EvaluationRequest) (*policyservice.EvaluationResponse, error) { + c.captured = req + if c.err != nil { + return nil, c.err + } + if c.response != nil { + return c.response, nil + } + return &policyservice.EvaluationResponse{}, nil +} + +// TestServiceClient_Evaluate_ThreadsAgentInfo guards against the exact bug +// this change fixed: an AgentInfo field (Name/Environment/ServiceTypes/Cost) +// silently dropped on the way from the placement/policy adapter into +// policyservice.EvaluationRequest. +func TestServiceClient_Evaluate_ThreadsAgentInfo(t *testing.T) { + eval := &capturingEvaluationService{} + client := NewServiceClient(eval) + + req := EvaluateRequest{ + Spec: map[string]any{"service_type": "vm"}, + AvailableAgents: []AgentInfo{ + {Name: "agent-a", Environment: "prod", ServiceTypes: []string{"vm", "database"}, Cost: "low"}, + }, + ExcludeAgents: []string{"agent-b"}, + } + + _, err := client.Evaluate(context.Background(), req) + if err != nil { + t.Fatalf("Evaluate returned unexpected error: %v", err) + } + + if eval.captured == nil { + t.Fatal("EvaluationService was not called") + } + if len(eval.captured.AvailableAgents) != 1 { + t.Fatalf("AvailableAgents = %d agents, want 1", len(eval.captured.AvailableAgents)) + } + got := eval.captured.AvailableAgents[0] + want := policyservice.AgentInfo{Name: "agent-a", Environment: "prod", ServiceTypes: []string{"vm", "database"}, Cost: "low"} + if got.Name != want.Name || got.Environment != want.Environment || got.Cost != want.Cost { + t.Errorf("AgentInfo = %+v, want %+v", got, want) + } + if len(got.ServiceTypes) != 2 || got.ServiceTypes[0] != "vm" || got.ServiceTypes[1] != "database" { + t.Errorf("ServiceTypes = %v, want [vm database]", got.ServiceTypes) + } + if len(eval.captured.ExcludeAgents) != 1 || eval.captured.ExcludeAgents[0] != "agent-b" { + t.Errorf("ExcludeAgents = %v, want [agent-b]", eval.captured.ExcludeAgents) + } +} + +func TestServiceClient_Evaluate_MapsErrorOnFailure(t *testing.T) { + eval := &capturingEvaluationService{err: policyservice.NewNoCapableAgentError("database", 0)} + client := NewServiceClient(eval) + + _, err := client.Evaluate(context.Background(), EvaluateRequest{Spec: map[string]any{}}) + + var httpErr *HTTPError + if !errors.As(err, &httpErr) { + t.Fatalf("expected *HTTPError, got %T: %v", err, err) + } + if httpErr.StatusCode != http.StatusNotAcceptable { + t.Errorf("StatusCode = %d, want %d", httpErr.StatusCode, http.StatusNotAcceptable) + } +} diff --git a/internal/placement/policy/types.go b/internal/placement/policy/types.go index 8ef87bc..f9ef609 100644 --- a/internal/placement/policy/types.go +++ b/internal/placement/policy/types.go @@ -6,16 +6,29 @@ import ( "fmt" ) +// AgentInfo is the subset of agent metadata passed through to policy +// evaluation. Kept as a parallel type rather than importing +// internal/policy/service.AgentInfo directly, per this package's convention +// of not depending on that package's types. +type AgentInfo struct { + Name string `json:"name"` + Environment string `json:"environment"` + ServiceTypes []string `json:"service_types,omitempty"` + Cost string `json:"cost,omitempty"` +} + // EvaluateRequest is the input for policy evaluation. type EvaluateRequest struct { - Spec map[string]any `json:"spec"` + Spec map[string]any `json:"spec"` + AvailableAgents []AgentInfo `json:"available_agents,omitempty"` + ExcludeAgents []string `json:"exclude_agents,omitempty"` } // EvaluateResponse is the result of policy evaluation. type EvaluateResponse struct { - Status string `json:"status"` - SelectedProvider string `json:"selected_provider"` - EvaluatedSpec map[string]any `json:"evaluated_spec"` + Status string `json:"status"` + SelectedAgent string `json:"selected_agent"` + EvaluatedSpec map[string]any `json:"evaluated_spec"` } // Client is the port PlacementService uses to evaluate policies. diff --git a/internal/placement/service/convert.go b/internal/placement/service/convert.go index 84e4fe2..a86d723 100644 --- a/internal/placement/service/convert.go +++ b/internal/placement/service/convert.go @@ -26,7 +26,7 @@ func storeModelToResource(m *model.Resource) types.Resource { RequiresResources: append([]string(nil), m.RequiresResources...), DagLevel: m.DagLevel, Status: m.Status, - ProviderName: m.ProviderName, + AgentName: m.AgentName, ApprovalStatus: m.ApprovalStatus, CreateTime: PtrTime(m.CreateTime), UpdateTime: PtrTime(m.UpdateTime), diff --git a/internal/placement/service/errors.go b/internal/placement/service/errors.go index 0a9e4b7..90fe3a5 100644 --- a/internal/placement/service/errors.go +++ b/internal/placement/service/errors.go @@ -3,6 +3,7 @@ package service import ( "errors" "fmt" + "log/slog" "net/http" "github.com/dcm-project/control-plane/internal/placement/policy" @@ -14,13 +15,14 @@ const ( ErrCodeNotFound = "https://dcm.example.com/errors/not-found" ErrCodeConflict = "https://dcm.example.com/errors/conflict" ErrCodeValidation = "https://dcm.example.com/errors/validation" - ErrCodeProviderError = "https://dcm.example.com/errors/provider-error" + ErrCodeProvisioningError = "https://dcm.example.com/errors/provisioning-error" ErrCodeInternal = "https://dcm.example.com/errors/internal-error" ErrCodePolicyError = "https://dcm.example.com/errors/policy-error" ErrCodePolicyInternalError = "https://dcm.example.com/errors/policy-internal-error" ErrCodePolicyRejected = "https://dcm.example.com/errors/policy-rejected" ErrCodePolicyConflict = "https://dcm.example.com/errors/policy-conflict" ErrCodeSPRMError = "https://dcm.example.com/errors/sprm-error" + ErrCodeUnavailable = "https://dcm.example.com/errors/unavailable" ) // ServiceError represents a business logic error with a code for HTTP mapping. @@ -98,6 +100,13 @@ func NewSPRMError(message string) *ServiceError { } } +func NewUnavailableError(message string) *ServiceError { + return &ServiceError{ + Code: ErrCodeUnavailable, + Message: message, + } +} + // IsClientError returns true if err is a ServiceError representing a client-side // (4xx) problem. If svcErr is non-nil it is populated with the unwrapped error. func IsClientError(err error, svcErr **ServiceError) bool { @@ -106,7 +115,7 @@ func IsClientError(err error, svcErr **ServiceError) bool { } switch (*svcErr).Code { case ErrCodeValidation, ErrCodeNotFound, ErrCodeConflict, - ErrCodePolicyRejected, ErrCodePolicyConflict, ErrCodeProviderError: + ErrCodePolicyRejected, ErrCodePolicyConflict, ErrCodeProvisioningError: return true } return false @@ -137,7 +146,9 @@ func handlePolicyError(err error) *ServiceError { } // handleSPRMError maps SPRM client errors to service errors by checking -// the error type and extracting the HTTP status code. +// the error type and extracting the HTTP status code. 5xx bodies are logged +// server-side and replaced with a generic client-facing message so internal +// error detail (DB errors, NATS errors) never leaks into API responses. func handleSPRMError(err error) *ServiceError { var httpErr *sprm.HTTPError if errors.As(err, &httpErr) { @@ -150,15 +161,21 @@ func handleSPRMError(err error) *ServiceError { return NewConflictError(fmt.Sprintf("resource conflict in SPRM: %s", httpErr.Body)) case http.StatusUnprocessableEntity: return &ServiceError{ - Code: ErrCodeProviderError, - Message: fmt.Sprintf("SPRM provider error: %s", httpErr.Body), + Code: ErrCodeProvisioningError, + Message: fmt.Sprintf("SPRM provisioning error: %s", httpErr.Body), } + case http.StatusServiceUnavailable: + slog.Error("sprm request failed", "status", httpErr.StatusCode, "detail", httpErr.Body) + return NewUnavailableError("service temporarily unavailable") case http.StatusInternalServerError: - return NewSPRMError(fmt.Sprintf("SPRM internal error: %s", httpErr.Body)) + slog.Error("sprm request failed", "status", httpErr.StatusCode, "detail", httpErr.Body) + return NewSPRMError("internal server error") default: - return NewSPRMError(fmt.Sprintf("SPRM request failed with status %d: %s", httpErr.StatusCode, httpErr.Body)) + slog.Error("sprm request failed", "status", httpErr.StatusCode, "detail", httpErr.Body) + return NewSPRMError("internal server error") } } - return NewSPRMError("SPRM request failed: " + err.Error()) + slog.Error("sprm request failed", "error", err) + return NewSPRMError("internal server error") } diff --git a/internal/placement/service/placement.go b/internal/placement/service/placement.go index 8503767..fa10009 100644 --- a/internal/placement/service/placement.go +++ b/internal/placement/service/placement.go @@ -9,6 +9,7 @@ import ( "slices" "time" + placementagent "github.com/dcm-project/control-plane/internal/placement/agent" "github.com/dcm-project/control-plane/internal/placement/logging" "github.com/dcm-project/control-plane/internal/placement/policy" "github.com/dcm-project/control-plane/internal/placement/sprm" @@ -22,18 +23,31 @@ const resourceRollbackTimeout = 10 * time.Second // PlacementService handles business logic for placement request management. type PlacementService struct { - store store.Store - policy policy.Client - sprm sprm.Client + store store.Store + policy policy.Client + sprm sprm.Client + agentClient placementagent.Client } // NewPlacementService creates a new PlacementService with the given store, policy client, and SPRM client. -func NewPlacementService(store store.Store, policyClient policy.Client, sprmClient sprm.Client) *PlacementService { - return &PlacementService{ +func NewPlacementService(store store.Store, policyClient policy.Client, sprmClient sprm.Client, opts ...func(*PlacementService)) *PlacementService { + ps := &PlacementService{ store: store, policy: policyClient, sprm: sprmClient, } + for _, opt := range opts { + opt(ps) + } + return ps +} + +// WithAgentClient sets the agent client used to list ready agents for +// policy evaluation. +func WithAgentClient(client placementagent.Client) func(*PlacementService) { + return func(ps *PlacementService) { + ps.agentClient = client + } } // CreateRun executes a placement run for one or more resources. @@ -63,6 +77,12 @@ func (s *PlacementService) CreateRun(ctx context.Context, req *types.CreateRunRe evaluatedSpec map[string]any } + availableAgents, err := s.listAvailableAgents(ctx) + if err != nil { + log.Error("Failed to list available agents", "error", err) + return nil, err + } + // step 3: evaluate policy for each resource prepared := make([]preparedResource, 0, len(req.Resources)) for _, resource := range req.Resources { @@ -72,25 +92,28 @@ func (s *PlacementService) CreateRun(ctx context.Context, req *types.CreateRunRe path := fmt.Sprintf("resources/%s", resourceID) // Evaluate spec with policy engine - policyRequest := policy.EvaluateRequest{Spec: resource.Spec} + policyRequest := policy.EvaluateRequest{ + Spec: resource.Spec, + AvailableAgents: availableAgents, + } log.Debug("Evaluating policy", "run_id", runID, "resource_id", resourceID, "name", resource.Name) policyResponse, err := s.policy.Evaluate(ctx, policyRequest) if err != nil { log.Error("Policy evaluation failed", "run_id", runID, "resource_id", resourceID, "error", err) return nil, handlePolicyError(err) } - if policyResponse.SelectedProvider == "" { - log.Error("Policy response missing selected provider", + if policyResponse.SelectedAgent == "" { + log.Error("Policy response missing selected agent", "run_id", runID, "resource_id", resourceID, "status", policyResponse.Status, ) - return nil, NewPolicyInternalError("policy response missing selected provider") + return nil, NewPolicyInternalError("policy response missing selected agent") } - // Extract approvalStatus and providerName from policy response + // Extract approvalStatus and agentName from policy response approval := policyResponse.Status - provider := policyResponse.SelectedProvider + agentName := policyResponse.SelectedAgent prepared = append(prepared, preparedResource{ resource: model.Resource{ @@ -103,7 +126,7 @@ func (s *PlacementService) CreateRun(ctx context.Context, req *types.CreateRunRe DagLevel: resourceNameDagLevelMap[resource.Name], Status: types.ResourceStatusPending, Path: path, - ProviderName: &provider, + AgentName: &agentName, ApprovalStatus: &approval, }, evaluatedSpec: policyResponse.EvaluatedSpec, @@ -133,9 +156,9 @@ func (s *PlacementService) CreateRun(ctx context.Context, req *types.CreateRunRe continue } sprmRequest := sprm.CreateResourceRequest{ - ID: p.resource.ID, - Spec: p.evaluatedSpec, - ProviderName: *p.resource.ProviderName, + ID: p.resource.ID, + Spec: p.evaluatedSpec, + AgentName: *p.resource.AgentName, } log.Debug("Provisioning resource via SPRM", "run_id", runID, @@ -318,7 +341,16 @@ func (s *PlacementService) RehydrateResource(ctx context.Context, runID, newRunI newResourceID := uuid.New().String() // Step 2: Re-evaluate the original spec through policy - policyRequest := policy.EvaluateRequest{Spec: oldResource.Spec} + availableAgents, err := s.listAvailableAgents(ctx) + if err != nil { + log.Error("Failed to list available agents for rehydration", "error", err) + return nil, err + } + policyRequest := policy.EvaluateRequest{ + Spec: oldResource.Spec, + AvailableAgents: availableAgents, + } + log.Debug("Re-evaluating policy for rehydration", "resource_id", resourceID) policyResponse, err := s.policy.Evaluate(ctx, policyRequest) if err != nil { @@ -326,17 +358,16 @@ func (s *PlacementService) RehydrateResource(ctx context.Context, runID, newRunI return nil, handlePolicyError(err) } - if policyResponse.SelectedProvider == "" { - log.Error("Policy response missing selected provider during rehydration", + if policyResponse.SelectedAgent == "" { + log.Error("Policy response missing selected agent during rehydration", "resource_id", resourceID, "status", policyResponse.Status, ) - return nil, NewPolicyInternalError("policy response missing selected provider") + return nil, NewPolicyInternalError("policy response missing selected agent") } - // Extract approvalStatus and providerName from policy response + // Extract approvalStatus from policy response approvalStatus := policyResponse.Status - providerName := policyResponse.SelectedProvider // Step 3: Create new resource in DB newPath := fmt.Sprintf("resources/%s", newResourceID) @@ -350,8 +381,8 @@ func (s *PlacementService) RehydrateResource(ctx context.Context, runID, newRunI DagLevel: oldResource.DagLevel, Status: types.ResourceStatusPending, Path: newPath, - ProviderName: &providerName, ApprovalStatus: &approvalStatus, + AgentName: &policyResponse.SelectedAgent, } // Step 3: Create new resource in DB @@ -367,9 +398,9 @@ func (s *PlacementService) RehydrateResource(ctx context.Context, runID, newRunI // Step 4: Provision new resource in SPRM sprmRequest := sprm.CreateResourceRequest{ - ID: newResourceID, - Spec: policyResponse.EvaluatedSpec, - ProviderName: providerName, + ID: newResourceID, + Spec: policyResponse.EvaluatedSpec, + AgentName: policyResponse.SelectedAgent, } if _, err = s.sprm.CreateResource(ctx, sprmRequest); err != nil { log.Error("SPRM provisioning failed during rehydration, rolling back", "new_resource_id", newResourceID, "error", err) @@ -407,7 +438,7 @@ func (s *PlacementService) RehydrateResource(ctx context.Context, runID, newRunI "old_resource_id", resourceID, "new_resource_id", newResourceID, "catalog_item_instance_id", oldResource.CatalogItemInstanceId, - "provider", providerName, + "agent", policyResponse.SelectedAgent, "approval_status", approvalStatus, ) @@ -444,6 +475,174 @@ func (s *PlacementService) rollbackResourceDelete(id string) error { return s.store.Resource().Delete(rbCtx, id) } +// ReEvaluateWithExclude re-evaluates placement for an existing resource, +// excluding the given agents (typically an agent that just failed or timed +// out), and re-provisions the resource against the newly selected agent. +// This is the core of the self-healing loop invoked by the pending/queued +// sweep: it does not create a new resource, it re-points the existing one. +// +// It also proactively reassigns any run-sibling still pointed at an +// excluded agent, best-effort, instead of leaving it to wait for its own +// independent sweep timeout. +func (s *PlacementService) ReEvaluateWithExclude(ctx context.Context, resourceID string, excludeAgents []string) error { + log := logging.FromContext(ctx) + + resource, err := s.store.Resource().Get(ctx, resourceID) + if err != nil { + if errors.Is(err, store.ErrResourceNotFound) { + return NewNotFoundError(fmt.Sprintf("resource %s not found", resourceID)) + } + return NewInternalError(fmt.Sprintf("failed to get resource: %v", err)) + } + + availableAgents, err := s.listAvailableAgents(ctx) + if err != nil { + log.Error("Failed to list available agents for re-evaluation", "error", err) + return err + } + + // The agent this resource was on when the caller (the pending/queued + // sweep) decided to reassign it: see reassignOne's expectedCurrentAgent + // doc for why this must be the caller's observation, not a fresh read. + var expectedCurrentAgent string + if resource.AgentName != nil { + expectedCurrentAgent = *resource.AgentName + } + + newAgent, err := s.reassignOne(ctx, resourceID, resource.Spec, expectedCurrentAgent, excludeAgents, availableAgents) + if err != nil { + return err + } + log.Info("Resource re-evaluated and reassigned", "resource_id", resourceID, "new_agent", newAgent) + + s.reassignExcludedSiblings(ctx, resource, excludeAgents, availableAgents) + + return nil +} + +// reassignOne evaluates policy for a single resource excluding +// excludeAgents and, on success, reassigns it in SPRM and persists the new +// agent_name. Shared by ReEvaluateWithExclude for the primary resource and +// by reassignExcludedSiblings for its run-siblings. +// +// expectedCurrentAgent is the agent this resource was observed on by the +// caller (primary: the resource's own record; sibling: the sibling's own +// record) at decision time. It's passed through to SPRM/SP unchanged so the +// eventual CAS there guards against the exact race this function runs +// concurrently with: another healer reassigning the same resource/instance +// between this function's policy evaluation and its own reassignment call. +func (s *PlacementService) reassignOne(ctx context.Context, resourceID string, spec map[string]any, expectedCurrentAgent string, excludeAgents []string, availableAgents []policy.AgentInfo) (string, error) { + log := logging.FromContext(ctx) + + policyResponse, err := s.policy.Evaluate(ctx, policy.EvaluateRequest{ + Spec: spec, + ExcludeAgents: excludeAgents, + AvailableAgents: availableAgents, + }) + if err != nil { + log.Error("Re-evaluation failed", "resource_id", resourceID, "error", err) + return "", handlePolicyError(err) + } + + if policyResponse.SelectedAgent == "" { + return "", NewPolicyInternalError("re-evaluation found no available agent") + } + + // Defensive re-check: nothing stops a misbehaving policy from returning + // an excluded agent anyway, which would reassign the instance right + // back to the agent it was just excluded to avoid. + for _, excluded := range excludeAgents { + if policyResponse.SelectedAgent == excluded { + return "", NewPolicyInternalError(fmt.Sprintf("re-evaluation selected excluded agent %q", excluded)) + } + } + + if err := s.sprm.ReassignResource(ctx, resourceID, policyResponse.SelectedAgent, expectedCurrentAgent); err != nil { + log.Error("Failed to reassign resource to new agent", "resource_id", resourceID, "new_agent", policyResponse.SelectedAgent, "error", err) + return "", handleSPRMError(err) + } + + // Propagated rather than swallowed: leaving this silent would let + // Resource.agent_name go stale, and retrying is safe since this is + // idempotent (ReassignAndReset is a CAS). + if err := s.store.Resource().UpdateAgentName(ctx, resourceID, policyResponse.SelectedAgent); err != nil { + log.Error("Failed to update resource agent_name after re-evaluation", "resource_id", resourceID, "error", err) + return "", NewInternalError(fmt.Sprintf("failed to update resource agent_name: %v", err)) + } + + return policyResponse.SelectedAgent, nil +} + +// reassignExcludedSiblings proactively reassigns run-siblings of resource +// that are still pointed at an excluded agent. Best-effort: a sibling +// failure is logged and skipped, never propagated to the caller, since the +// primary resource's own reassignment (already done by the time this runs) +// is what the self-heal loop depends on for its retry decision. +// +// reassignOne's underlying CAS (ReassignAndReset) only accepts +// pending/cancelled instances, so a sibling that's actively +// provisioning/running is automatically left alone. A queued sibling is +// also left alone here — its cancel-then-heal transition is handled by +// sweepQueued on its own timeout, which this deliberately doesn't +// replicate to keep this change bounded. +func (s *PlacementService) reassignExcludedSiblings(ctx context.Context, resource *model.Resource, excludeAgents []string, availableAgents []policy.AgentInfo) { + log := logging.FromContext(ctx) + + // Goes through the public GetRun rather than the store directly: it's + // the same run-scoped fetch either way (resource is already a known + // member of this RunID, so GetRun's zero-resources NotFoundError can't + // trigger here), and this is what the reviewer explicitly asked for. + run, err := s.GetRun(ctx, resource.RunID) + if err != nil { + log.Error("Failed to get run for proactive sibling reassignment", "run_id", resource.RunID, "error", err) + return + } + + for _, sibling := range run.Resources { + siblingID := "" + if sibling.Id != nil { + siblingID = *sibling.Id + } + if siblingID == resource.ID || sibling.AgentName == nil || !slices.Contains(excludeAgents, *sibling.AgentName) { + continue + } + newAgent, err := s.reassignOne(ctx, siblingID, sibling.Spec, *sibling.AgentName, excludeAgents, availableAgents) + if err != nil { + log.Warn("Best-effort sibling reassignment failed, will retry on its own sweep timeout", + "resource_id", siblingID, "run_id", resource.RunID, "error", err) + continue + } + log.Info("Run-sibling proactively reassigned", "resource_id", siblingID, "run_id", resource.RunID, "new_agent", newAgent) + } +} + +// listAvailableAgents returns the current ready agents for policy evaluation, +// or nil if no agent client is configured. A listing failure is returned as a +// hard error rather than degrading to an empty slice: ConstraintContext and +// EvaluatePolicies treat an empty AvailableAgents as "skip membership/ +// environment validation", so silently falling back there would turn an +// operational error into a fail-open placement decision. +func (s *PlacementService) listAvailableAgents(ctx context.Context) ([]policy.AgentInfo, error) { + if s.agentClient == nil { + return nil, nil + } + agents, err := s.agentClient.ListReadyAgents(ctx) + if err != nil { + return nil, NewInternalError(fmt.Sprintf("failed to list available agents: %v", err)) + } + return toPolicyAgentInfo(agents), nil +} + +// toPolicyAgentInfo maps the placement/agent package's Info to the parallel +// policy.AgentInfo used by policy.EvaluateRequest. +func toPolicyAgentInfo(agents []placementagent.Info) []policy.AgentInfo { + out := make([]policy.AgentInfo, len(agents)) + for i, a := range agents { + out[i] = policy.AgentInfo{Name: a.Name, Environment: a.Environment, ServiceTypes: a.ServiceTypes, Cost: a.Cost} + } + return out +} + func getOrGenerateStringId(id *string) string { if id != nil && *id != "" { return *id diff --git a/internal/placement/service/placement_test.go b/internal/placement/service/placement_test.go index b81af43..8fc898d 100644 --- a/internal/placement/service/placement_test.go +++ b/internal/placement/service/placement_test.go @@ -4,6 +4,8 @@ import ( "context" "errors" + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" + placementagent "github.com/dcm-project/control-plane/internal/placement/agent" "github.com/dcm-project/control-plane/internal/placement/policy" "github.com/dcm-project/control-plane/internal/placement/service" "github.com/dcm-project/control-plane/internal/placement/sprm" @@ -30,9 +32,9 @@ func (m *mockPolicyClient) Evaluate(ctx context.Context, req policy.EvaluateRequ } // Default: approve with the original spec return &policy.EvaluateResponse{ - Status: "APPROVED", - SelectedProvider: "default-provider", - EvaluatedSpec: req.Spec, + Status: "APPROVED", + SelectedAgent: "default-agent", + EvaluatedSpec: req.Spec, }, nil } @@ -41,6 +43,7 @@ type mockSPRMClient struct { CreateResourceFunc func(ctx context.Context, req sprm.CreateResourceRequest) (*sprm.CreateResourceResponse, error) DeleteResourceFunc func(ctx context.Context, resourceId string) error DeleteResourceDeferredFunc func(ctx context.Context, resourceId string) error + ReassignResourceFunc func(ctx context.Context, resourceId string, agentName string, expectedCurrentAgent string) error } // CreateResource calls the mock function if set, otherwise returns a default success response @@ -71,6 +74,26 @@ func (m *mockSPRMClient) DeleteResourceDeferred(ctx context.Context, resourceId return nil } +// ReassignResource calls the mock function if set, otherwise returns success +func (m *mockSPRMClient) ReassignResource(ctx context.Context, resourceId string, agentName string, expectedCurrentAgent string) error { + if m.ReassignResourceFunc != nil { + return m.ReassignResourceFunc(ctx, resourceId, agentName, expectedCurrentAgent) + } + return nil +} + +type mockAgentClient struct { + agents []placementagent.Info + err error +} + +func (m *mockAgentClient) ListReadyAgents(_ context.Context) ([]placementagent.Info, error) { + if m.err != nil { + return nil, m.err + } + return m.agents, nil +} + func getStoredResource(ctx context.Context, dataStore store.Store, id string) *model.Resource { r, err := dataStore.Resource().Get(ctx, id) Expect(err).NotTo(HaveOccurred()) @@ -102,6 +125,7 @@ var _ = Describe("PlacementService", func() { dataStore store.Store mockPolicy *mockPolicyClient mockSPRM *mockSPRMClient + agentClient *mockAgentClient placementSvc *service.PlacementService ctx context.Context ) @@ -112,12 +136,20 @@ var _ = Describe("PlacementService", func() { Logger: logger.Default.LogMode(logger.Silent), }) Expect(err).NotTo(HaveOccurred()) - Expect(db.AutoMigrate(&model.Resource{})).To(Succeed()) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.Resource{})).To(Succeed()) + + for _, name := range []string{"default-agent", "test-agent", "modified-agent", "async-agent", "new-agent", "fallback-agent"} { + Expect(db.Create(&agentmodel.Agent{ID: uuid.New().String(), Name: name, TopicName: "dcm.agent." + name}).Error).NotTo(HaveOccurred()) + } dataStore = store.NewStore(db) mockPolicy = &mockPolicyClient{} mockSPRM = &mockSPRMClient{} - placementSvc = service.NewPlacementService(dataStore, mockPolicy, mockSPRM) + agentClient = &mockAgentClient{agents: []placementagent.Info{ + {Name: "agent-a", Environment: "prod", ServiceTypes: []string{"vm"}, Cost: "low"}, + {Name: "agent-b", Environment: "prod", ServiceTypes: []string{"vm"}, Cost: "medium"}, + }} + placementSvc = service.NewPlacementService(dataStore, mockPolicy, mockSPRM, service.WithAgentClient(agentClient)) ctx = context.Background() }) @@ -127,12 +159,12 @@ var _ = Describe("PlacementService", func() { }) Describe("CreateRun", func() { - It("creates resource with APPROVED status from policy", func() { + It("creates resource with APPROVED status and agent routing", func() { mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { return &policy.EvaluateResponse{ - Status: "APPROVED", - SelectedProvider: "test-provider", - EvaluatedSpec: req.Spec, + Status: "APPROVED", + SelectedAgent: "test-agent", + EvaluatedSpec: req.Spec, }, nil } @@ -153,15 +185,14 @@ var _ = Describe("PlacementService", func() { Expect(result.Resources[0].Spec).To(HaveKey("memory")) Expect(result.Resources[0].ApprovalStatus).NotTo(BeNil()) Expect(*result.Resources[0].ApprovalStatus).To(Equal("APPROVED")) - Expect(result.Resources[0].ProviderName).NotTo(BeNil()) - Expect(*result.Resources[0].ProviderName).To(Equal("test-provider")) + Expect(result.Resources[0].AgentName).NotTo(BeNil()) + Expect(*result.Resources[0].AgentName).To(Equal("test-agent")) - stored, err := dataStore.Resource().Get(ctx, *result.Resources[0].Id) - Expect(err).NotTo(HaveOccurred()) + stored := getStoredResource(ctx, dataStore, *result.Resources[0].Id) Expect(stored.ApprovalStatus).NotTo(BeNil()) Expect(*stored.ApprovalStatus).To(Equal("APPROVED")) - Expect(stored.ProviderName).NotTo(BeNil()) - Expect(*stored.ProviderName).To(Equal("test-provider")) + Expect(stored.AgentName).NotTo(BeNil()) + Expect(*stored.AgentName).To(Equal("test-agent")) }) It("rejects empty run_id", func() { @@ -190,7 +221,7 @@ var _ = Describe("PlacementService", func() { Expect(svcErr.Code).To(Equal(service.ErrCodeConflict)) }) - It("creates resource with MODIFIED status from policy", func() { + It("creates resource with MODIFIED status and agent routing", func() { mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { modifiedSpec := make(map[string]any) for k, v := range req.Spec { @@ -198,9 +229,9 @@ var _ = Describe("PlacementService", func() { } modifiedSpec["modified_field"] = "policy_value" return &policy.EvaluateResponse{ - Status: "MODIFIED", - SelectedProvider: "modified-provider", - EvaluatedSpec: modifiedSpec, + Status: "MODIFIED", + SelectedAgent: "modified-agent", + EvaluatedSpec: modifiedSpec, }, nil } @@ -217,14 +248,14 @@ var _ = Describe("PlacementService", func() { Expect(result.Resources[0].Spec).NotTo(HaveKey("modified_field")) // Original spec preserved Expect(result.Resources[0].ApprovalStatus).NotTo(BeNil()) Expect(*result.Resources[0].ApprovalStatus).To(Equal("MODIFIED")) - Expect(*result.Resources[0].ProviderName).To(Equal("modified-provider")) + Expect(result.Resources[0].AgentName).NotTo(BeNil()) + Expect(*result.Resources[0].AgentName).To(Equal("modified-agent")) - stored, err := dataStore.Resource().Get(ctx, *result.Resources[0].Id) - Expect(err).NotTo(HaveOccurred()) + stored := getStoredResource(ctx, dataStore, *result.Resources[0].Id) Expect(stored.ApprovalStatus).NotTo(BeNil()) Expect(*stored.ApprovalStatus).To(Equal("MODIFIED")) - Expect(stored.ProviderName).NotTo(BeNil()) - Expect(*stored.ProviderName).To(Equal("modified-provider")) + Expect(stored.AgentName).NotTo(BeNil()) + Expect(*stored.AgentName).To(Equal("modified-agent")) }) It("creates resource with specified ID", func() { @@ -241,6 +272,51 @@ var _ = Describe("PlacementService", func() { Expect(*result.Resources[0].Id).To(Equal(specifiedID)) }) + It("passes available agents to policy", func() { + var capturedReq policy.EvaluateRequest + mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + capturedReq = req + return &policy.EvaluateResponse{ + Status: "APPROVED", + SelectedAgent: "test-agent", + EvaluatedSpec: req.Spec, + }, nil + } + + resource := &types.Resource{ + CatalogItemInstanceId: "cat-avail-agents", + Spec: map[string]any{"service_type": "vm"}, + } + _, err := placementSvc.CreateRun(ctx, singleResourceRun(resource.CatalogItemInstanceId, resource.Spec, nil)) + Expect(err).NotTo(HaveOccurred()) + + Expect(capturedReq.AvailableAgents).NotTo(BeEmpty()) + names := make([]string, 0, len(capturedReq.AvailableAgents)) + for _, a := range capturedReq.AvailableAgents { + names = append(names, a.Name) + } + Expect(names).To(ContainElement("agent-a")) + Expect(names).To(ContainElement("agent-b")) + }) + + It("fails closed and does not evaluate policy when listing available agents errors", func() { + agentClient.err = errors.New("agent registry unavailable") + policyCalled := false + mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + policyCalled = true + return &policy.EvaluateResponse{Status: "APPROVED", SelectedAgent: "test-agent", EvaluatedSpec: req.Spec}, nil + } + + result, err := placementSvc.CreateRun(ctx, singleResourceRun("catalog-agents-error", map[string]any{"cpu": 1}, nil)) + + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + Expect(policyCalled).To(BeFalse()) + var svcErr *service.ServiceError + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeInternal)) + }) + It("returns error when policy validation fails (400)", func() { mockPolicy.EvaluateFunc = func(_ context.Context, _ policy.EvaluateRequest) (*policy.EvaluateResponse, error) { return nil, &policy.HTTPError{StatusCode: 400, Body: "bad request"} @@ -343,17 +419,17 @@ var _ = Describe("PlacementService", func() { Expect(svcErr.Message).To(ContainSubstring("I'm a teapot")) }) - It("returns error when policy response is missing selected provider", func() { + It("returns error when policy response is missing selected agent", func() { mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { return &policy.EvaluateResponse{ - Status: "APPROVED", - SelectedProvider: "", - EvaluatedSpec: req.Spec, + Status: "APPROVED", + SelectedAgent: "", + EvaluatedSpec: req.Spec, }, nil } resource := &types.Resource{ - CatalogItemInstanceId: "catalog-no-provider", + CatalogItemInstanceId: "catalog-no-agent", Spec: map[string]any{"cpu": 2}, } @@ -365,7 +441,7 @@ var _ = Describe("PlacementService", func() { Expect(err).To(BeAssignableToTypeOf(svcErr)) svcErr = err.(*service.ServiceError) Expect(svcErr.Code).To(Equal(service.ErrCodePolicyInternalError)) - Expect(svcErr.Message).To(ContainSubstring("missing selected provider")) + Expect(svcErr.Message).To(ContainSubstring("missing selected agent")) }) It("returns error when policy client communication fails", func() { @@ -485,9 +561,9 @@ var _ = Describe("PlacementService", func() { Expect(resources.Resources).To(BeEmpty()) }) - It("returns provider error when SPRM creation fails (422)", func() { + It("returns provisioning error when SPRM creation fails (422)", func() { mockSPRM.CreateResourceFunc = func(_ context.Context, _ sprm.CreateResourceRequest) (*sprm.CreateResourceResponse, error) { - return nil, &sprm.HTTPError{StatusCode: 422, Body: "provider validation failed"} + return nil, &sprm.HTTPError{StatusCode: 422, Body: "agent validation failed"} } resource := &types.Resource{ @@ -502,13 +578,42 @@ var _ = Describe("PlacementService", func() { var svcErr *service.ServiceError Expect(err).To(BeAssignableToTypeOf(svcErr)) svcErr = err.(*service.ServiceError) - Expect(svcErr.Code).To(Equal(service.ErrCodeProviderError)) + Expect(svcErr.Code).To(Equal(service.ErrCodeProvisioningError)) // Verify resource was NOT persisted in DB (rollback worked) resources, err := dataStore.Resource().ListRun(ctx, &store.ResourceListOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(resources.Resources).To(BeEmpty()) }) + + It("does not rollback on 202 from SPRM", func() { + mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + return &policy.EvaluateResponse{ + Status: "APPROVED", + SelectedAgent: "async-agent", + EvaluatedSpec: req.Spec, + }, nil + } + mockSPRM.CreateResourceFunc = func(_ context.Context, req sprm.CreateResourceRequest) (*sprm.CreateResourceResponse, error) { + return &sprm.CreateResourceResponse{ + ID: req.ID, + Status: "accepted", + }, nil + } + + resource := &types.Resource{ + CatalogItemInstanceId: "cat-202", + Spec: map[string]any{"service_type": "vm"}, + } + + result, err := placementSvc.CreateRun(ctx, singleResourceRun(resource.CatalogItemInstanceId, resource.Spec, nil)) + + Expect(err).NotTo(HaveOccurred()) + Expect(result).NotTo(BeNil()) + + stored := getStoredResource(ctx, dataStore, *result.Resources[0].Id) + Expect(stored).NotTo(BeNil()) + }) }) Describe("DeleteRun", func() { @@ -642,9 +747,9 @@ var _ = Describe("PlacementService", func() { mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { return &policy.EvaluateResponse{ - Status: "APPROVED", - SelectedProvider: "test-provider", - EvaluatedSpec: req.Spec, + Status: "APPROVED", + SelectedAgent: "test-agent", + EvaluatedSpec: req.Spec, }, nil } @@ -671,7 +776,8 @@ var _ = Describe("PlacementService", func() { Expect(result.Spec).To(HaveKey("cpu")) Expect(result.Spec).To(HaveKey("memory")) Expect(*result.ApprovalStatus).To(Equal("APPROVED")) - Expect(*result.ProviderName).To(Equal("test-provider")) + Expect(result.AgentName).NotTo(BeNil()) + Expect(*result.AgentName).To(Equal("test-agent")) // Verify old resource is gone expectStoredResourceMissing(ctx, dataStore, oldResourceID) @@ -679,21 +785,46 @@ var _ = Describe("PlacementService", func() { stored := getStoredResource(ctx, dataStore, *result.Id) Expect(stored.CatalogItemInstanceId).To(Equal(catalogID)) Expect(stored.RunID).To(Equal(newRunID)) + Expect(stored.AgentName).NotTo(BeNil()) + Expect(*stored.AgentName).To(Equal("test-agent")) }) - It("re-evaluates policy and assigns new provider", func() { + It("re-evaluates policy and assigns new agent", func() { mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { return &policy.EvaluateResponse{ - Status: "APPROVED", - SelectedProvider: "new-provider", - EvaluatedSpec: req.Spec, + Status: "APPROVED", + SelectedAgent: "new-agent", + EvaluatedSpec: req.Spec, }, nil } - result, err := placementSvc.RehydrateResource(ctx, oldRunID, "new-run-provider") + result, err := placementSvc.RehydrateResource(ctx, oldRunID, "new-run-agent") Expect(err).NotTo(HaveOccurred()) - Expect(*result.ProviderName).To(Equal("new-provider")) + stored := getStoredResource(ctx, dataStore, *result.Id) + Expect(stored.AgentName).NotTo(BeNil()) + Expect(*stored.AgentName).To(Equal("new-agent")) + }) + + It("fails closed and does not re-evaluate policy when listing available agents errors", func() { + agentClient.err = errors.New("agent registry unavailable") + policyCalled := false + mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + policyCalled = true + return &policy.EvaluateResponse{Status: "APPROVED", SelectedAgent: "test-agent", EvaluatedSpec: req.Spec}, nil + } + + result, err := placementSvc.RehydrateResource(ctx, oldRunID, "new-run-agents-error") + + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + Expect(policyCalled).To(BeFalse()) + var svcErr *service.ServiceError + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeInternal)) + + // Old resource must survive an aborted rehydration. + _ = getStoredResource(ctx, dataStore, oldResourceID) }) It("preserves original spec and sends evaluated spec to SPRM", func() { @@ -705,9 +836,9 @@ var _ = Describe("PlacementService", func() { } modifiedSpec["policy_added"] = "value" return &policy.EvaluateResponse{ - Status: "MODIFIED", - SelectedProvider: "test-provider", - EvaluatedSpec: modifiedSpec, + Status: "MODIFIED", + SelectedAgent: "test-agent", + EvaluatedSpec: modifiedSpec, }, nil } mockSPRM.CreateResourceFunc = func(_ context.Context, req sprm.CreateResourceRequest) (*sprm.CreateResourceResponse, error) { @@ -791,12 +922,12 @@ var _ = Describe("PlacementService", func() { _ = getStoredResource(ctx, dataStore, oldResourceID) }) - It("returns error when policy returns empty provider", func() { + It("returns error when policy returns empty agent", func() { mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { return &policy.EvaluateResponse{ - Status: "APPROVED", - SelectedProvider: "", - EvaluatedSpec: req.Spec, + Status: "APPROVED", + SelectedAgent: "", + EvaluatedSpec: req.Spec, }, nil } @@ -883,4 +1014,362 @@ var _ = Describe("PlacementService", func() { Expect(page2.NextPageToken).To(BeNil()) }) }) + + Describe("ReEvaluateWithExclude", func() { + It("re-evaluates with excluded agent, calls SPRM to reassign, and persists the new agent", func() { + resource := &types.Resource{ + CatalogItemInstanceId: "cat-reeval-1", + Spec: map[string]any{"service_type": "vm"}, + } + created, err := placementSvc.CreateRun(ctx, singleResourceRun(resource.CatalogItemInstanceId, resource.Spec, nil)) + Expect(err).NotTo(HaveOccurred()) + resourceID := *created.Resources[0].Id + Expect(*getStoredResource(ctx, dataStore, resourceID).AgentName).To(Equal("default-agent")) + + var evalReq policy.EvaluateRequest + mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + evalReq = req + return &policy.EvaluateResponse{ + Status: "APPROVED", + SelectedAgent: "fallback-agent", + EvaluatedSpec: map[string]any{}, + }, nil + } + + var reassignedID, reassignedAgent, reassignedExpectedCurrent string + mockSPRM.ReassignResourceFunc = func(_ context.Context, resourceId string, agentName string, expectedCurrentAgent string) error { + reassignedID = resourceId + reassignedAgent = agentName + reassignedExpectedCurrent = expectedCurrentAgent + return nil + } + + err = placementSvc.ReEvaluateWithExclude(ctx, resourceID, []string{"failed-agent"}) + + Expect(err).NotTo(HaveOccurred()) + Expect(evalReq.ExcludeAgents).To(ConsistOf("failed-agent")) + Expect(reassignedID).To(Equal(resourceID)) + Expect(reassignedAgent).To(Equal("fallback-agent")) + // The CAS-critical value: must be the resource's own + // pre-reassignment agent ("default-agent" from CreateRun), not + // the excluded agent (which need not be the same in general) + // and not re-derived at SPRM/SP call time. + Expect(reassignedExpectedCurrent).To(Equal("default-agent")) + + stored := getStoredResource(ctx, dataStore, resourceID) + Expect(stored.AgentName).NotTo(BeNil()) + Expect(*stored.AgentName).To(Equal("fallback-agent")) + }) + + It("returns error when no viable agent remains", func() { + resource := &types.Resource{ + CatalogItemInstanceId: "cat-no-agent", + Spec: map[string]any{"service_type": "vm"}, + } + + created, err := placementSvc.CreateRun(ctx, singleResourceRun(resource.CatalogItemInstanceId, resource.Spec, nil)) + Expect(err).NotTo(HaveOccurred()) + resourceID := *created.Resources[0].Id + + mockPolicy.EvaluateFunc = func(_ context.Context, _ policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + return nil, &policy.HTTPError{StatusCode: 404, Body: "no agents"} + } + + err = placementSvc.ReEvaluateWithExclude(ctx, resourceID, []string{"all-agents"}) + + Expect(err).To(HaveOccurred()) + + stored := getStoredResource(ctx, dataStore, resourceID) + Expect(*stored.AgentName).To(Equal("default-agent")) + }) + + It("returns error and leaves the resource unchanged when SPRM reassignment fails", func() { + resource := &types.Resource{ + CatalogItemInstanceId: "cat-reeval-fail", + Spec: map[string]any{"service_type": "vm"}, + } + created, err := placementSvc.CreateRun(ctx, singleResourceRun(resource.CatalogItemInstanceId, resource.Spec, nil)) + Expect(err).NotTo(HaveOccurred()) + resourceID := *created.Resources[0].Id + Expect(*getStoredResource(ctx, dataStore, resourceID).AgentName).To(Equal("default-agent")) + + mockPolicy.EvaluateFunc = func(_ context.Context, _ policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + return &policy.EvaluateResponse{ + Status: "APPROVED", + SelectedAgent: "fallback-agent", + EvaluatedSpec: map[string]any{}, + }, nil + } + mockSPRM.ReassignResourceFunc = func(_ context.Context, _ string, _ string, _ string) error { + return &sprm.HTTPError{StatusCode: 503, Body: "sprm unavailable"} + } + + err = placementSvc.ReEvaluateWithExclude(ctx, resourceID, []string{"failed-agent"}) + + Expect(err).To(HaveOccurred()) + var svcErr *service.ServiceError + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeUnavailable)) + + stored := getStoredResource(ctx, dataStore, resourceID) + Expect(*stored.AgentName).To(Equal("default-agent")) + }) + + It("fails closed and does not re-evaluate policy when listing available agents errors", func() { + resource := &types.Resource{ + CatalogItemInstanceId: "cat-reeval-agents-error", + Spec: map[string]any{"service_type": "vm"}, + } + created, err := placementSvc.CreateRun(ctx, singleResourceRun(resource.CatalogItemInstanceId, resource.Spec, nil)) + Expect(err).NotTo(HaveOccurred()) + resourceID := *created.Resources[0].Id + + agentClient.err = errors.New("agent registry unavailable") + policyCalled := false + mockPolicy.EvaluateFunc = func(_ context.Context, _ policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + policyCalled = true + return &policy.EvaluateResponse{Status: "APPROVED", SelectedAgent: "fallback-agent", EvaluatedSpec: map[string]any{}}, nil + } + + err = placementSvc.ReEvaluateWithExclude(ctx, resourceID, []string{"failed-agent"}) + + Expect(err).To(HaveOccurred()) + Expect(policyCalled).To(BeFalse()) + var svcErr *service.ServiceError + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeInternal)) + + stored := getStoredResource(ctx, dataStore, resourceID) + Expect(*stored.AgentName).To(Equal("default-agent")) + }) + + It("returns not found when resource does not exist", func() { + err := placementSvc.ReEvaluateWithExclude(ctx, "non-existent", []string{"failed-agent"}) + + Expect(err).To(HaveOccurred()) + var svcErr *service.ServiceError + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeNotFound)) + }) + + It("rejects a policy response that selects an excluded agent instead of reassigning to it (R2 S3: finding #7)", func() { + // Defensive check: nothing besides the Rego policy itself + // enforces exclude_agents, so a misbehaving/misconfigured policy + // could hand back the very agent the caller asked to avoid. Must + // be rejected rather than silently reassigning the instance + // right back to the failing agent. + resource := &types.Resource{ + CatalogItemInstanceId: "cat-reeval-excluded", + Spec: map[string]any{"service_type": "vm"}, + } + created, err := placementSvc.CreateRun(ctx, singleResourceRun(resource.CatalogItemInstanceId, resource.Spec, nil)) + Expect(err).NotTo(HaveOccurred()) + resourceID := *created.Resources[0].Id + + mockPolicy.EvaluateFunc = func(_ context.Context, _ policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + return &policy.EvaluateResponse{ + Status: "APPROVED", + SelectedAgent: "failed-agent", + EvaluatedSpec: map[string]any{}, + }, nil + } + reassignCalled := false + mockSPRM.ReassignResourceFunc = func(_ context.Context, _ string, _ string, _ string) error { + reassignCalled = true + return nil + } + + err = placementSvc.ReEvaluateWithExclude(ctx, resourceID, []string{"failed-agent"}) + + Expect(err).To(HaveOccurred()) + var svcErr *service.ServiceError + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(reassignCalled).To(BeFalse()) + + stored := getStoredResource(ctx, dataStore, resourceID) + Expect(*stored.AgentName).To(Equal("default-agent")) + }) + + It("propagates a failure to persist the new agent_name instead of silently leaving it stale (R2 S3: finding #10)", func() { + resource := &types.Resource{ + CatalogItemInstanceId: "cat-reeval-persist-fail", + Spec: map[string]any{"service_type": "vm"}, + } + created, err := placementSvc.CreateRun(ctx, singleResourceRun(resource.CatalogItemInstanceId, resource.Spec, nil)) + Expect(err).NotTo(HaveOccurred()) + resourceID := *created.Resources[0].Id + + mockPolicy.EvaluateFunc = func(_ context.Context, _ policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + return &policy.EvaluateResponse{ + Status: "APPROVED", + SelectedAgent: "fallback-agent", + EvaluatedSpec: map[string]any{}, + }, nil + } + // SPRM/sp-side reassignment succeeds, but the resource row backing + // placement's own agent_name is deleted as a side effect (simulating + // e.g. a concurrent delete), so the subsequent UpdateAgentName finds + // zero rows and fails. + mockSPRM.ReassignResourceFunc = func(_ context.Context, resourceId string, _ string, _ string) error { + Expect(db.Delete(&model.Resource{}, "id = ?", resourceId).Error).NotTo(HaveOccurred()) + return nil + } + + err = placementSvc.ReEvaluateWithExclude(ctx, resourceID, []string{"failed-agent"}) + + Expect(err).To(HaveOccurred()) + var svcErr *service.ServiceError + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeInternal)) + }) + + // Multi-resource run-sibling proactive reassignment (thread 9 upgrade): + // a sibling still pointed at the excluded agent gets reassigned + // alongside the primary resource, instead of waiting for its own + // independent sweep timeout. + createSiblingRun := func(catalogID string, primarySpec, siblingSpec map[string]any) (primaryID, siblingID string) { + mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + return &policy.EvaluateResponse{Status: "APPROVED", SelectedAgent: "failed-agent", EvaluatedSpec: req.Spec}, nil + } + created, err := placementSvc.CreateRun(ctx, &types.CreateRunRequest{ + CatalogItemInstanceId: catalogID, + RunId: uuid.New().String(), + Resources: []types.ResourceInput{ + {Name: "primary", Spec: primarySpec}, + {Name: "sibling", Spec: siblingSpec}, + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(created.Resources).To(HaveLen(2)) + return *created.Resources[0].Id, *created.Resources[1].Id + } + + It("proactively reassigns a run-sibling stuck on the excluded agent", func() { + primaryID, siblingID := createSiblingRun("cat-reeval-sibling-pending", + map[string]any{"service_type": "vm"}, map[string]any{"service_type": "db"}) + + var evaluatedSpecs []map[string]any + mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + evaluatedSpecs = append(evaluatedSpecs, req.Spec) + return &policy.EvaluateResponse{Status: "APPROVED", SelectedAgent: "fallback-agent", EvaluatedSpec: req.Spec}, nil + } + reassigned := map[string]string{} + expectedCurrentByID := map[string]string{} + mockSPRM.ReassignResourceFunc = func(_ context.Context, resourceId string, agentName string, expectedCurrentAgent string) error { + reassigned[resourceId] = agentName + expectedCurrentByID[resourceId] = expectedCurrentAgent + return nil + } + + err := placementSvc.ReEvaluateWithExclude(ctx, primaryID, []string{"failed-agent"}) + + Expect(err).NotTo(HaveOccurred()) + Expect(reassigned).To(HaveKeyWithValue(primaryID, "fallback-agent")) + Expect(reassigned).To(HaveKeyWithValue(siblingID, "fallback-agent")) + Expect(*getStoredResource(ctx, dataStore, siblingID).AgentName).To(Equal("fallback-agent")) + // Each resource's own excluded agent is what gets CASed, not a + // value shared across the primary/sibling reassignment calls. + Expect(expectedCurrentByID).To(HaveKeyWithValue(primaryID, "failed-agent")) + Expect(expectedCurrentByID).To(HaveKeyWithValue(siblingID, "failed-agent")) + + specs := make([]any, len(evaluatedSpecs)) + for i, s := range evaluatedSpecs { + specs[i] = s["service_type"] + } + Expect(specs).To(ContainElement("db")) + }) + + It("leaves a sibling untouched when it is not CAS-eligible for reassignment (provisioning/running/queued)", func() { + primaryID, siblingID := createSiblingRun("cat-reeval-sibling-ineligible", + map[string]any{"service_type": "vm"}, map[string]any{"service_type": "db"}) + + mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + return &policy.EvaluateResponse{Status: "APPROVED", SelectedAgent: "fallback-agent", EvaluatedSpec: req.Spec}, nil + } + reassigned := map[string]string{} + mockSPRM.ReassignResourceFunc = func(_ context.Context, resourceId string, agentName string, _ string) error { + if resourceId == siblingID { + // Mirrors ReassignAndReset's CAS rejecting a sibling that + // isn't pending/cancelled sp-side (e.g. provisioning, + // running, or queued on the excluded agent). + return &sprm.HTTPError{StatusCode: 409, Body: "instance is not eligible for reassignment"} + } + reassigned[resourceId] = agentName + return nil + } + + err := placementSvc.ReEvaluateWithExclude(ctx, primaryID, []string{"failed-agent"}) + + Expect(err).NotTo(HaveOccurred()) + Expect(reassigned).To(HaveKeyWithValue(primaryID, "fallback-agent")) + Expect(reassigned).NotTo(HaveKey(siblingID)) + Expect(*getStoredResource(ctx, dataStore, siblingID).AgentName).To(Equal("failed-agent")) + }) + + It("leaves a sibling on a different, non-excluded agent alone", func() { + // A 3-resource run where only one sibling shares the excluded + // agent with the primary; the other was already routed + // elsewhere and must not be touched. + mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + agent := "failed-agent" + if req.Spec["service_type"] == "other" { + agent = "healthy-agent" + } + return &policy.EvaluateResponse{Status: "APPROVED", SelectedAgent: agent, EvaluatedSpec: req.Spec}, nil + } + created, err := placementSvc.CreateRun(ctx, &types.CreateRunRequest{ + CatalogItemInstanceId: "cat-reeval-sibling-mixed", + RunId: uuid.New().String(), + Resources: []types.ResourceInput{ + {Name: "primary", Spec: map[string]any{"service_type": "vm"}}, + {Name: "sibling-same-agent", Spec: map[string]any{"service_type": "db"}}, + {Name: "sibling-other-agent", Spec: map[string]any{"service_type": "other"}}, + }, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(created.Resources).To(HaveLen(3)) + primaryID, siblingSameID, siblingOtherID := *created.Resources[0].Id, *created.Resources[1].Id, *created.Resources[2].Id + + mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + return &policy.EvaluateResponse{Status: "APPROVED", SelectedAgent: "fallback-agent", EvaluatedSpec: req.Spec}, nil + } + reassigned := map[string]string{} + mockSPRM.ReassignResourceFunc = func(_ context.Context, resourceId string, agentName string, _ string) error { + reassigned[resourceId] = agentName + return nil + } + + err = placementSvc.ReEvaluateWithExclude(ctx, primaryID, []string{"failed-agent"}) + + Expect(err).NotTo(HaveOccurred()) + Expect(reassigned).To(HaveKeyWithValue(primaryID, "fallback-agent")) + Expect(reassigned).To(HaveKeyWithValue(siblingSameID, "fallback-agent")) + Expect(reassigned).NotTo(HaveKey(siblingOtherID)) + Expect(*getStoredResource(ctx, dataStore, siblingOtherID).AgentName).To(Equal("healthy-agent")) + }) + + It("still succeeds for the primary resource when a sibling's own re-evaluation fails (best-effort isolation)", func() { + primaryID, siblingID := createSiblingRun("cat-reeval-sibling-policy-fail", + map[string]any{"service_type": "vm"}, map[string]any{"service_type": "db"}) + + mockPolicy.EvaluateFunc = func(_ context.Context, req policy.EvaluateRequest) (*policy.EvaluateResponse, error) { + if req.Spec["service_type"] == "db" { + return nil, &policy.HTTPError{StatusCode: 404, Body: "no agents for db"} + } + return &policy.EvaluateResponse{Status: "APPROVED", SelectedAgent: "fallback-agent", EvaluatedSpec: req.Spec}, nil + } + reassigned := map[string]string{} + mockSPRM.ReassignResourceFunc = func(_ context.Context, resourceId string, agentName string, _ string) error { + reassigned[resourceId] = agentName + return nil + } + + err := placementSvc.ReEvaluateWithExclude(ctx, primaryID, []string{"failed-agent"}) + + Expect(err).NotTo(HaveOccurred()) + Expect(reassigned).To(HaveKeyWithValue(primaryID, "fallback-agent")) + Expect(reassigned).NotTo(HaveKey(siblingID)) + Expect(*getStoredResource(ctx, dataStore, siblingID).AgentName).To(Equal("failed-agent")) + }) + }) }) diff --git a/internal/placement/sprm/service_client.go b/internal/placement/sprm/service_client.go index 3a92241..8c597c2 100644 --- a/internal/placement/sprm/service_client.go +++ b/internal/placement/sprm/service_client.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "log/slog" "net/http" sprmv1alpha1 "github.com/dcm-project/control-plane/api/sp/v1alpha1/resource_manager" @@ -22,12 +23,11 @@ func NewServiceClient(instances *rmsvc.InstanceService) Client { func (c *serviceClient) CreateResource(ctx context.Context, req CreateResourceRequest) (*CreateResourceResponse, error) { body := sprmv1alpha1.ServiceTypeInstance{ - ProviderName: req.ProviderName, - Spec: req.Spec, + Spec: req.Spec, } queryID := req.ID - instance, err := c.instances.CreateInstance(ctx, &body, &queryID) + instance, err := c.instances.CreateInstance(ctx, &body, &queryID, req.AgentName) if err != nil { return nil, mapInstanceError(err) } @@ -57,6 +57,13 @@ func (c *serviceClient) deleteResource(ctx context.Context, resourceID string, d return nil } +func (c *serviceClient) ReassignResource(ctx context.Context, resourceID string, agentName string, expectedCurrentAgent string) error { + if err := c.instances.ReassignAgent(ctx, resourceID, agentName, expectedCurrentAgent); err != nil { + return mapInstanceError(err) + } + return nil +} + func mapInstanceError(err error) error { var svcErr *spservice.ServiceError if !errors.As(err, &svcErr) { @@ -71,9 +78,24 @@ func mapInstanceError(err error) error { status = http.StatusNotFound case spservice.ErrCodeConflict: status = http.StatusConflict - case spservice.ErrCodeProviderError: + case spservice.ErrCodeProvisioningError: status = http.StatusUnprocessableEntity + case spservice.ErrCodeUnavailable: + status = http.StatusServiceUnavailable + } + + // 4xx bodies are client-facing validation detail and safe to return + // verbatim. 5xx bodies may contain internal error strings (DB, NATS) - + // log them server-side and return a generic message to the caller. + body := svcErr.Message + if status >= http.StatusInternalServerError { + slog.Error("sprm adapter error", "status", status, "detail", svcErr.Message) + if status == http.StatusServiceUnavailable { + body = "service temporarily unavailable" + } else { + body = "internal server error" + } } - return &HTTPError{StatusCode: status, Body: svcErr.Message} + return &HTTPError{StatusCode: status, Body: body} } diff --git a/internal/placement/sprm/service_client_test.go b/internal/placement/sprm/service_client_test.go new file mode 100644 index 0000000..5bc95ce --- /dev/null +++ b/internal/placement/sprm/service_client_test.go @@ -0,0 +1,89 @@ +package sprm + +import ( + "errors" + "net/http" + "testing" + + spservice "github.com/dcm-project/control-plane/internal/sp/service" +) + +// TestMapInstanceError checks both the status code mapping per +// service.ErrorCode and that 5xx bodies never leak the internal error message. +func TestMapInstanceError(t *testing.T) { + cases := []struct { + name string + err error + wantStatusCode int + wantBodyExact string // "" means "don't check exact body" + wantBodyLeaks bool + }{ + { + name: "validation error maps to 400 and passes message through", + err: spservice.NewValidationError("spec.service_type is required"), + wantStatusCode: http.StatusBadRequest, + wantBodyExact: "spec.service_type is required", + }, + { + name: "not found error maps to 404 and passes message through", + err: spservice.NewNotFoundError("instance abc not found"), + wantStatusCode: http.StatusNotFound, + wantBodyExact: "instance abc not found", + }, + { + name: "conflict error maps to 409 and passes message through", + err: spservice.NewConflictError("instance abc is being deleted"), + wantStatusCode: http.StatusConflict, + wantBodyExact: "instance abc is being deleted", + }, + { + name: "provisioning error maps to 422 and passes message through", + err: spservice.NewProvisioningError("failed to publish create request"), + wantStatusCode: http.StatusUnprocessableEntity, + wantBodyExact: "failed to publish create request", + }, + { + name: "unavailable error maps to 503 with a sanitized body", + err: spservice.NewUnavailableError("nats publisher unavailable: dial tcp 10.0.0.1:4222: connect: connection refused"), + wantStatusCode: http.StatusServiceUnavailable, + wantBodyExact: "service temporarily unavailable", + }, + { + name: "internal error maps to 500 with a sanitized body, not the raw DB/NATS error", + err: spservice.NewInternalError("failed to retrieve instance: pq: connection reset by peer"), + wantStatusCode: http.StatusInternalServerError, + wantBodyExact: "internal server error", + }, + { + name: "a non-ServiceError is wrapped rather than mapped to an HTTPError", + err: errors.New("some unrelated infra error"), + wantStatusCode: 0, // sentinel: expect no *HTTPError at all + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := mapInstanceError(tc.err) + + var httpErr *HTTPError + ok := errors.As(got, &httpErr) + + if tc.wantStatusCode == 0 { + if ok { + t.Fatalf("expected a non-HTTPError wrapped error, got *HTTPError{%d, %q}", httpErr.StatusCode, httpErr.Body) + } + return + } + + if !ok { + t.Fatalf("expected *HTTPError, got %T: %v", got, got) + } + if httpErr.StatusCode != tc.wantStatusCode { + t.Errorf("StatusCode = %d, want %d", httpErr.StatusCode, tc.wantStatusCode) + } + if tc.wantBodyExact != "" && httpErr.Body != tc.wantBodyExact { + t.Errorf("Body = %q, want %q", httpErr.Body, tc.wantBodyExact) + } + }) + } +} diff --git a/internal/placement/sprm/types.go b/internal/placement/sprm/types.go index 0eaec38..e3237c2 100644 --- a/internal/placement/sprm/types.go +++ b/internal/placement/sprm/types.go @@ -8,9 +8,9 @@ import ( // CreateResourceRequest is the input for creating a service type instance. type CreateResourceRequest struct { - ID string `json:"id"` - Spec map[string]any `json:"spec"` - ProviderName string `json:"provider_name"` + ID string `json:"id"` + Spec map[string]any `json:"spec"` + AgentName string `json:"agent_name,omitempty"` } // CreateResourceResponse is the result of creating a service type instance. @@ -24,6 +24,16 @@ type Client interface { CreateResource(ctx context.Context, req CreateResourceRequest) (*CreateResourceResponse, error) DeleteResource(ctx context.Context, resourceId string) error DeleteResourceDeferred(ctx context.Context, resourceId string) error + // ReassignResource re-points an existing resource at a new agent and + // re-triggers provisioning. Used by the self-healing loop. + // + // expectedCurrentAgent must be the agent the caller observed this + // resource on when it decided to reassign it (e.g. the excluded/failed + // agent), not a value re-read at call time: it's CASed against the + // live agent_name at the SP layer specifically to catch a concurrent + // reassignment that happened in between, so re-deriving it fresh here + // would silently defeat that guard. + ReassignResource(ctx context.Context, resourceId string, agentName string, expectedCurrentAgent string) error } // HTTPError carries a status code and message from the SPRM adapter. diff --git a/internal/placement/store/model/resource.go b/internal/placement/store/model/resource.go index 368130d..907b6dd 100644 --- a/internal/placement/store/model/resource.go +++ b/internal/placement/store/model/resource.go @@ -6,6 +6,14 @@ import ( ) // Resource represents a resource provisioning request within a placement run. +// +// AgentName is a soft reference to an agent by name, not a GORM-level +// foreign key: an earlier version declared one with constraint:OnDelete:RESTRICT, +// which meant an agent could never be deregistered/deleted while any +// resource (including old, terminal ones) still referenced its name - the +// same class of problem addressed for ServiceTypeInstance.AgentName (F4). +// Agent existence/health is validated at the application layer instead +// (see placement/service), same as the sp domain. type Resource struct { ID string `gorm:"primaryKey;type:varchar(63)"` RunID string `gorm:"column:run_id;type:varchar(63);index;not null"` @@ -15,7 +23,7 @@ type Resource struct { RequiresResources []string `gorm:"column:requires_resources;type:jsonb;serializer:json"` DagLevel int `gorm:"column:dag_level;not null;default:0"` Status string `gorm:"column:status;type:varchar(63);not null;default:PENDING"` - ProviderName *string `gorm:"column:provider_name;not null"` + AgentName *string `gorm:"column:agent_name"` ApprovalStatus *string `gorm:"column:approval_status;not null"` Path string `gorm:"column:path;not null"` CreateTime time.Time `gorm:"column:create_time;autoCreateTime"` diff --git a/internal/placement/store/resource.go b/internal/placement/store/resource.go index fd80e4f..220cdf1 100644 --- a/internal/placement/store/resource.go +++ b/internal/placement/store/resource.go @@ -17,9 +17,9 @@ var ( // ResourceListOptions contains optional fields for listing runs. type ResourceListOptions struct { - ProviderName *string - PageSize int - PageToken *string + AgentName *string + PageSize int + PageToken *string } // ResourceListResult contains resources for a page of runs (complete sets per run_id). @@ -40,6 +40,7 @@ type Resource interface { DeleteByRunID(ctx context.Context, runID string) error UpdateRunID(ctx context.Context, oldRunID, newRunID string) error UpdateStatusByRunID(ctx context.Context, runID, status string) error + UpdateAgentName(ctx context.Context, id string, agentName string) error } type ResourceStore struct { @@ -71,8 +72,8 @@ func (s *ResourceStore) ListRun(ctx context.Context, opts *ResourceListOptions) query := s.db.WithContext(ctx).Model(&model.Resource{}) // Apply filters - if opts != nil && opts.ProviderName != nil && *opts.ProviderName != "" { - query = query.Where("provider_name = ?", *opts.ProviderName) + if opts != nil && opts.AgentName != nil && strings.TrimSpace(*opts.AgentName) != "" { + query = query.Where("agent_name = ?", *opts.AgentName) } // Page distinct run_ids (limit+1 to detect if there are more results). @@ -207,3 +208,16 @@ func (s *ResourceStore) UpdateStatusByRunID(ctx context.Context, runID, status s } return nil } + +// UpdateAgentName updates the agent_name column for observability after the +// self-healing loop re-routes a resource to a different agent. +func (s *ResourceStore) UpdateAgentName(ctx context.Context, id string, agentName string) error { + result := s.db.WithContext(ctx).Model(&model.Resource{}).Where("id = ?", id).Update("agent_name", agentName) + if result.Error != nil { + return result.Error + } + if result.RowsAffected == 0 { + return ErrResourceNotFound + } + return nil +} diff --git a/internal/placement/store/resource_test.go b/internal/placement/store/resource_test.go index d591dda..b30cf31 100644 --- a/internal/placement/store/resource_test.go +++ b/internal/placement/store/resource_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" "github.com/dcm-project/control-plane/internal/placement/store" "github.com/dcm-project/control-plane/internal/placement/store/model" "github.com/google/uuid" @@ -27,7 +28,7 @@ var _ = Describe("Resource Store", func() { Logger: logger.Default.LogMode(logger.Silent), }) Expect(err).NotTo(HaveOccurred()) - Expect(db.AutoMigrate(&model.Resource{})).To(Succeed()) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.Resource{})).To(Succeed()) requestStore = store.NewResource(db) ctx = context.Background() @@ -40,7 +41,7 @@ var _ = Describe("Resource Store", func() { Describe("Create", func() { It("persists the resource without optional fields", func() { - provider := "test-provider" + agent := "test-agent" approval := "APPROVED" r := model.Resource{ ID: uuid.New().String(), @@ -48,7 +49,7 @@ var _ = Describe("Resource Store", func() { Name: "main", CatalogItemInstanceId: "catalog-instance-123", Spec: map[string]any{"cpu": "2", "memory": "4Gi"}, - ProviderName: &provider, + AgentName: &agent, ApprovalStatus: &approval, Path: "resources/" + uuid.New().String(), } @@ -58,15 +59,15 @@ var _ = Describe("Resource Store", func() { Expect(created.ID).To(Equal(r.ID)) Expect(created.CatalogItemInstanceId).To(Equal("catalog-instance-123")) Expect(created.Spec).To(Equal(map[string]any{"cpu": "2", "memory": "4Gi"})) - Expect(created.ProviderName).NotTo(BeNil()) - Expect(*created.ProviderName).To(Equal("test-provider")) + Expect(created.AgentName).NotTo(BeNil()) + Expect(*created.AgentName).To(Equal("test-agent")) Expect(created.ApprovalStatus).NotTo(BeNil()) Expect(*created.ApprovalStatus).To(Equal("APPROVED")) }) It("returns error for duplicate ID", func() { id := uuid.New().String() - provider := "test-provider" + agent := "test-agent" approval := "APPROVED" r1 := model.Resource{ ID: id, @@ -74,7 +75,7 @@ var _ = Describe("Resource Store", func() { Name: "main", CatalogItemInstanceId: "catalog-instance-123", Spec: map[string]any{"cpu": "2"}, - ProviderName: &provider, + AgentName: &agent, ApprovalStatus: &approval, Path: "resources/" + id, } @@ -88,7 +89,7 @@ var _ = Describe("Resource Store", func() { Name: "main", CatalogItemInstanceId: "catalog-instance-456", Spec: map[string]any{"cpu": "4"}, - ProviderName: &provider, + AgentName: &agent, ApprovalStatus: &approval, Path: "resources/" + id, } @@ -100,7 +101,7 @@ var _ = Describe("Resource Store", func() { Describe("CreateBatch", func() { It("persists multiple resources in one call", func() { - provider := "test-provider" + agent := "test-agent" approval := "APPROVED" id1 := uuid.New().String() id2 := uuid.New().String() @@ -111,7 +112,7 @@ var _ = Describe("Resource Store", func() { Name: "db", CatalogItemInstanceId: "catalog-instance-123", Spec: map[string]any{"cpu": "2"}, - ProviderName: &provider, + AgentName: &agent, ApprovalStatus: &approval, Path: "resources/" + id1, DagLevel: 0, @@ -123,7 +124,7 @@ var _ = Describe("Resource Store", func() { CatalogItemInstanceId: "catalog-instance-123", Spec: map[string]any{"cpu": "4"}, RequiresResources: []string{"db"}, - ProviderName: &provider, + AgentName: &agent, ApprovalStatus: &approval, Path: "resources/" + id2, DagLevel: 1, @@ -142,7 +143,7 @@ var _ = Describe("Resource Store", func() { }) It("returns error when any resource ID already exists", func() { - provider := "test-provider" + agent := "test-agent" approval := "APPROVED" existingID := uuid.New().String() _, err := requestStore.Create(ctx, model.Resource{ @@ -151,7 +152,7 @@ var _ = Describe("Resource Store", func() { Name: "main", CatalogItemInstanceId: "catalog-instance-123", Spec: map[string]any{"cpu": "2"}, - ProviderName: &provider, + AgentName: &agent, ApprovalStatus: &approval, Path: "resources/" + existingID, }) @@ -165,7 +166,7 @@ var _ = Describe("Resource Store", func() { Name: "a", CatalogItemInstanceId: "catalog-instance-123", Spec: map[string]any{}, - ProviderName: &provider, + AgentName: &agent, ApprovalStatus: &approval, Path: "resources/" + newID, }, @@ -175,7 +176,7 @@ var _ = Describe("Resource Store", func() { Name: "b", CatalogItemInstanceId: "catalog-instance-123", Spec: map[string]any{}, - ProviderName: &provider, + AgentName: &agent, ApprovalStatus: &approval, Path: "resources/" + existingID, }, @@ -186,7 +187,7 @@ var _ = Describe("Resource Store", func() { Describe("Get", func() { It("retrieves by ID", func() { - provider := "test-provider" + agent := "test-agent" approval := "APPROVED" r := model.Resource{ ID: uuid.New().String(), @@ -194,7 +195,7 @@ var _ = Describe("Resource Store", func() { Name: "main", CatalogItemInstanceId: "catalog-instance-456", Spec: map[string]any{"test": "data"}, - ProviderName: &provider, + AgentName: &agent, ApprovalStatus: &approval, Path: "resources/" + uuid.New().String(), } @@ -215,19 +216,19 @@ var _ = Describe("Resource Store", func() { Describe("ListRun", func() { var ( - providerA = "provider-a" - providerB = "provider-b" - approval = "APPROVED" + agentA = "agent-a" + agentB = "agent-b" + approval = "APPROVED" ) - createResource := func(runID, name, provider, catalogID string) { + createResource := func(runID, name, agent, catalogID string) { _, err := requestStore.Create(ctx, model.Resource{ ID: uuid.New().String(), RunID: runID, Name: name, CatalogItemInstanceId: catalogID, Spec: map[string]any{}, - ProviderName: &provider, + AgentName: &agent, ApprovalStatus: &approval, Path: "resources/" + name, }) @@ -236,13 +237,13 @@ var _ = Describe("Resource Store", func() { BeforeEach(func() { // run-1: 2 resources (would span a page_size=2 resource list alone) - createResource("run-1", "db", providerA, "cat-1") - createResource("run-1", "app", providerA, "cat-1") - // run-2: 1 resource, different provider - createResource("run-2", "main", providerB, "cat-2") + createResource("run-1", "db", agentA, "cat-1") + createResource("run-1", "app", agentA, "cat-1") + // run-2: 1 resource, different agent + createResource("run-2", "main", agentB, "cat-2") // run-3: 2 resources - createResource("run-3", "db", providerA, "cat-3") - createResource("run-3", "app", providerA, "cat-3") + createResource("run-3", "db", agentA, "cat-3") + createResource("run-3", "app", agentA, "cat-3") }) It("paginates by run_id and returns complete resource sets", func() { @@ -275,28 +276,57 @@ var _ = Describe("Resource Store", func() { Expect(page1.NextPageToken).NotTo(BeNil()) }) - It("filters by provider name", func() { + It("filters by agent name", func() { page, err := requestStore.ListRun(ctx, &store.ResourceListOptions{ - ProviderName: &providerB, + AgentName: &agentB, }) Expect(err).NotTo(HaveOccurred()) Expect(page.Resources).To(HaveLen(1)) Expect(page.Resources[0].RunID).To(Equal("run-2")) - Expect(*page.Resources[0].ProviderName).To(Equal(providerB)) + Expect(*page.Resources[0].AgentName).To(Equal(agentB)) Expect(page.NextPageToken).To(BeNil()) }) - It("excludes other providers' resources within a mixed-provider run", func() { - createResource("run-mixed", "db", providerA, "cat-mixed") - createResource("run-mixed", "app", providerB, "cat-mixed") + It("excludes resources with no agent name when filtering by agent name", func() { + _, err := requestStore.Create(ctx, model.Resource{ + ID: uuid.New().String(), + RunID: "run-unassigned", + Name: "db", + CatalogItemInstanceId: "cat-unassigned", + Spec: map[string]any{}, + ApprovalStatus: &approval, + Path: "resources/db", + }) + Expect(err).NotTo(HaveOccurred()) + + page, err := requestStore.ListRun(ctx, &store.ResourceListOptions{ + AgentName: &agentB, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(page.Resources).To(HaveLen(1)) + Expect(page.Resources[0].RunID).To(Equal("run-2")) + }) + + It("treats a blank agent name as no filter", func() { + blank := " " + page, err := requestStore.ListRun(ctx, &store.ResourceListOptions{ + AgentName: &blank, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(page.Resources).To(HaveLen(5)) + }) + + It("excludes other agents' resources within a mixed-agent run", func() { + createResource("run-mixed", "db", agentA, "cat-mixed") + createResource("run-mixed", "app", agentB, "cat-mixed") page, err := requestStore.ListRun(ctx, &store.ResourceListOptions{ - ProviderName: &providerB, + AgentName: &agentB, }) Expect(err).NotTo(HaveOccurred()) Expect(page.Resources).To(HaveLen(2)) for _, r := range page.Resources { - Expect(*r.ProviderName).To(Equal(providerB)) + Expect(*r.AgentName).To(Equal(agentB)) } Expect(page.Resources[0].RunID).To(Equal("run-2")) Expect(page.Resources[1].RunID).To(Equal("run-mixed")) @@ -342,7 +372,7 @@ var _ = Describe("Resource Store", func() { Describe("Delete", func() { It("deletes the resource", func() { - provider := "test-provider" + agent := "test-agent" approval := "APPROVED" r := model.Resource{ ID: uuid.New().String(), @@ -350,7 +380,7 @@ var _ = Describe("Resource Store", func() { Name: "main", CatalogItemInstanceId: "cat-del", Spec: map[string]any{}, - ProviderName: &provider, + AgentName: &agent, ApprovalStatus: &approval, Path: "resources/del", } @@ -373,7 +403,7 @@ var _ = Describe("Resource Store", func() { Describe("UpdateStatusByRunID", func() { It("updates status for all resources in the run", func() { - provider := "test-provider" + agent := "test-agent" approval := "APPROVED" id1 := uuid.New().String() id2 := uuid.New().String() @@ -384,7 +414,7 @@ var _ = Describe("Resource Store", func() { Name: "db", CatalogItemInstanceId: "cat-1", Spec: map[string]any{}, - ProviderName: &provider, + AgentName: &agent, ApprovalStatus: &approval, Path: "resources/" + id1, Status: "PENDING", @@ -396,7 +426,7 @@ var _ = Describe("Resource Store", func() { Name: "app", CatalogItemInstanceId: "cat-1", Spec: map[string]any{}, - ProviderName: &provider, + AgentName: &agent, ApprovalStatus: &approval, Path: "resources/" + id2, Status: "PENDING", diff --git a/internal/placement/types/resource.go b/internal/placement/types/resource.go index d74b859..f6d101e 100644 --- a/internal/placement/types/resource.go +++ b/internal/placement/types/resource.go @@ -20,6 +20,7 @@ type CreateRunRequest struct { // Resource is a placement resource row within a run. type Resource struct { + AgentName *string `json:"agent_name,omitempty"` ApprovalStatus *string `json:"approval_status,omitempty"` CatalogItemInstanceId string `json:"catalog_item_instance_id"` CreateTime *time.Time `json:"create_time,omitempty"` @@ -27,7 +28,6 @@ type Resource struct { Id *string `json:"id,omitempty"` Name string `json:"name"` Path *string `json:"path,omitempty"` - ProviderName *string `json:"provider_name,omitempty"` RequiresResources []string `json:"requires_resources,omitempty"` RunId string `json:"run_id"` Spec map[string]any `json:"spec"` diff --git a/internal/policy/opa/evaluation.go b/internal/policy/opa/evaluation.go index 115a8e5..ce4646b 100644 --- a/internal/policy/opa/evaluation.go +++ b/internal/policy/opa/evaluation.go @@ -2,24 +2,25 @@ package opa // EvaluationResult represents the result from OPA evaluation type EvaluationResult struct { - Result map[string]any // The policy decision - Defined bool // Whether the policy made a decision + Result map[string]any + Defined bool } -// ServiceProviderConstraints represents constraints on which service providers are allowed -type ServiceProviderConstraints struct { - AllowList []string `json:"allow_list,omitempty"` - Patterns []string `json:"patterns,omitempty"` +// AgentConstraints represents constraints on which agents are allowed +type AgentConstraints struct { + AllowList []string `json:"allow_list,omitempty"` + Patterns []string `json:"patterns,omitempty"` + EnvironmentConstraints []string `json:"environment_constraints,omitempty"` } // PolicyDecision represents the expected output from OPA policies type PolicyDecision struct { - Rejected bool `json:"rejected"` - RejectionReason string `json:"rejection_reason,omitempty"` - Patch map[string]any `json:"patch,omitempty"` - Constraints map[string]any `json:"constraints,omitempty"` - ServiceProviderConstraints *ServiceProviderConstraints `json:"service_provider_constraints,omitempty"` - SelectedProvider string `json:"selected_provider,omitempty"` + Rejected bool `json:"rejected"` + RejectionReason string `json:"rejection_reason,omitempty"` + Patch map[string]any `json:"patch,omitempty"` + Constraints map[string]any `json:"constraints,omitempty"` + SelectedAgent string `json:"selected_agent,omitempty"` + AgentConstraints *AgentConstraints `json:"agent_constraints,omitempty"` } // ParsePolicyDecision extracts a PolicyDecision from the OPA evaluation result @@ -42,27 +43,34 @@ func ParsePolicyDecision(result map[string]any) *PolicyDecision { decision.Constraints = constraints } - if spc, ok := result["service_provider_constraints"].(map[string]any); ok { - spConstraints := &ServiceProviderConstraints{} - if allowList, ok := spc["allow_list"].([]any); ok { + if agent, ok := result["selected_agent"].(string); ok { + decision.SelectedAgent = agent + } + + if ac, ok := result["agent_constraints"].(map[string]any); ok { + agentConstraints := &AgentConstraints{} + if allowList, ok := ac["allow_list"].([]any); ok { for _, item := range allowList { if s, ok := item.(string); ok { - spConstraints.AllowList = append(spConstraints.AllowList, s) + agentConstraints.AllowList = append(agentConstraints.AllowList, s) } } } - if patterns, ok := spc["patterns"].([]any); ok { + if patterns, ok := ac["patterns"].([]any); ok { for _, item := range patterns { if s, ok := item.(string); ok { - spConstraints.Patterns = append(spConstraints.Patterns, s) + agentConstraints.Patterns = append(agentConstraints.Patterns, s) } } } - decision.ServiceProviderConstraints = spConstraints - } - - if provider, ok := result["selected_provider"].(string); ok { - decision.SelectedProvider = provider + if envConstraints, ok := ac["environment_constraints"].([]any); ok { + for _, item := range envConstraints { + if s, ok := item.(string); ok { + agentConstraints.EnvironmentConstraints = append(agentConstraints.EnvironmentConstraints, s) + } + } + } + decision.AgentConstraints = agentConstraints } return decision diff --git a/internal/policy/opa/evaluation_test.go b/internal/policy/opa/evaluation_test.go index 9ae6da7..7342ead 100644 --- a/internal/policy/opa/evaluation_test.go +++ b/internal/policy/opa/evaluation_test.go @@ -19,14 +19,14 @@ func TestParsePolicyDecision(t *testing.T) { "patch": map[string]interface{}{ "region": "us-east-1", }, - "selected_provider": "aws", + "selected_agent": "aws-agent", }, expected: &PolicyDecision{ Rejected: false, Patch: map[string]interface{}{ "region": "us-east-1", }, - SelectedProvider: "aws", + SelectedAgent: "aws-agent", }, }, { @@ -55,17 +55,17 @@ func TestParsePolicyDecision(t *testing.T) { }, }, { - name: "approval with service provider constraints", + name: "approval with agent constraints", result: map[string]interface{}{ "rejected": false, - "service_provider_constraints": map[string]interface{}{ + "agent_constraints": map[string]interface{}{ "allow_list": []interface{}{"aws", "gcp"}, "patterns": []interface{}{"^(aws|gcp)$"}, }, }, expected: &PolicyDecision{ Rejected: false, - ServiceProviderConstraints: &ServiceProviderConstraints{ + AgentConstraints: &AgentConstraints{ AllowList: []string{"aws", "gcp"}, Patterns: []string{"^(aws|gcp)$"}, }, @@ -113,13 +113,13 @@ func TestParsePolicyDecision(t *testing.T) { assert.Equal(t, tt.expected.RejectionReason, decision.RejectionReason) assert.Equal(t, tt.expected.Patch, decision.Patch) assert.Equal(t, tt.expected.Constraints, decision.Constraints) - assert.Equal(t, tt.expected.SelectedProvider, decision.SelectedProvider) - if tt.expected.ServiceProviderConstraints != nil { - assert.NotNil(t, decision.ServiceProviderConstraints) - assert.Equal(t, tt.expected.ServiceProviderConstraints.AllowList, decision.ServiceProviderConstraints.AllowList) - assert.Equal(t, tt.expected.ServiceProviderConstraints.Patterns, decision.ServiceProviderConstraints.Patterns) + assert.Equal(t, tt.expected.SelectedAgent, decision.SelectedAgent) + if tt.expected.AgentConstraints != nil { + assert.NotNil(t, decision.AgentConstraints) + assert.Equal(t, tt.expected.AgentConstraints.AllowList, decision.AgentConstraints.AllowList) + assert.Equal(t, tt.expected.AgentConstraints.Patterns, decision.AgentConstraints.Patterns) } else { - assert.Nil(t, decision.ServiceProviderConstraints) + assert.Nil(t, decision.AgentConstraints) } }) } diff --git a/internal/policy/service/agent_constraints_test.go b/internal/policy/service/agent_constraints_test.go new file mode 100644 index 0000000..22ca152 --- /dev/null +++ b/internal/policy/service/agent_constraints_test.go @@ -0,0 +1,113 @@ +package service + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("AccumulatedAgentConstraints", func() { + var constraintCtx *ConstraintContext + + BeforeEach(func() { + constraintCtx = NewConstraintContext() + }) + + Context("MergeAgentConstraints", func() { + It("merges allow lists by intersection", func() { + first := &AccumulatedAgentConstraints{ + AllowList: []string{"agent-a", "agent-b", "agent-c"}, + } + err := constraintCtx.MergeAgentConstraints(first, "policy-1") + Expect(err).NotTo(HaveOccurred()) + + second := &AccumulatedAgentConstraints{ + AllowList: []string{"agent-b", "agent-c", "agent-d"}, + } + err = constraintCtx.MergeAgentConstraints(second, "policy-2") + Expect(err).NotTo(HaveOccurred()) + + err = constraintCtx.ValidateAgent("agent-b", nil) + Expect(err).NotTo(HaveOccurred()) + + err = constraintCtx.ValidateAgent("agent-a", nil) + Expect(err).To(HaveOccurred()) + }) + + It("enforces tightening-only across priorities", func() { + first := &AccumulatedAgentConstraints{ + AllowList: []string{"agent-a"}, + } + err := constraintCtx.MergeAgentConstraints(first, "policy-1") + Expect(err).NotTo(HaveOccurred()) + + second := &AccumulatedAgentConstraints{ + AllowList: []string{"agent-a", "agent-b"}, + } + err = constraintCtx.MergeAgentConstraints(second, "policy-2") + Expect(err).NotTo(HaveOccurred()) + + err = constraintCtx.ValidateAgent("agent-b", nil) + Expect(err).To(HaveOccurred()) + }) + + It("validates agent name against constraints", func() { + constraints := &AccumulatedAgentConstraints{ + AllowList: []string{"allowed-agent"}, + Patterns: []string{"^allowed-.*"}, + } + err := constraintCtx.MergeAgentConstraints(constraints, "policy-1") + Expect(err).NotTo(HaveOccurred()) + + err = constraintCtx.ValidateAgent("allowed-agent", nil) + Expect(err).NotTo(HaveOccurred()) + + err = constraintCtx.ValidateAgent("denied-agent", nil) + Expect(err).To(HaveOccurred()) + }) + + It("validates agent environment against environment_constraints", func() { + constraints := &AccumulatedAgentConstraints{ + EnvironmentConstraints: []string{"production", "staging"}, + } + err := constraintCtx.MergeAgentConstraints(constraints, "policy-1") + Expect(err).NotTo(HaveOccurred()) + + err = constraintCtx.ValidateAgentEnvironment("production") + Expect(err).NotTo(HaveOccurred()) + + err = constraintCtx.ValidateAgentEnvironment("development") + Expect(err).To(HaveOccurred()) + }) + }) + + // ValidateAgent must reject a selected agent outside availableAgents + // even when no policy declared an allow-list at all. + Context("ValidateAgent against availableAgents (fail-fast on agent selection)", func() { + It("rejects a selected agent absent from availableAgents even with no other constraints set", func() { + err := constraintCtx.ValidateAgent("rogue-agent", []string{"agent-a", "agent-b"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("rogue-agent")) + }) + + It("accepts a selected agent present in availableAgents", func() { + err := constraintCtx.ValidateAgent("agent-a", []string{"agent-a", "agent-b"}) + Expect(err).NotTo(HaveOccurred()) + }) + + It("does not enforce the available-agents check when availableAgents is empty", func() { + err := constraintCtx.ValidateAgent("any-agent", nil) + Expect(err).NotTo(HaveOccurred()) + }) + + It("checks availableAgents membership before policy allow-list constraints", func() { + constraints := &AccumulatedAgentConstraints{ + AllowList: []string{"rogue-agent"}, // would pass the allow-list alone + } + err := constraintCtx.MergeAgentConstraints(constraints, "policy-1") + Expect(err).NotTo(HaveOccurred()) + + err = constraintCtx.ValidateAgent("rogue-agent", []string{"agent-a", "agent-b"}) + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/internal/policy/service/constraints.go b/internal/policy/service/constraints.go index 1c00cc4..5350969 100644 --- a/internal/policy/service/constraints.go +++ b/internal/policy/service/constraints.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/brunoga/deep/v4" - "github.com/dcm-project/control-plane/internal/policy/opa" "github.com/santhosh-tekuri/jsonschema/v6" ) @@ -17,14 +16,133 @@ import ( type ConstraintContext struct { constrainedFieldsByFieldPath map[string]map[string]any // field path → JSON Schema keywords policyIdByFieldPath map[string]string // field path → policy ID that set it - serviceProviderConstraints *AccumulatedSPConstraints + agentConstraints *AccumulatedAgentConstraints } -// AccumulatedSPConstraints tracks accumulated service provider constraints -type AccumulatedSPConstraints struct { - AllowList []string // Intersection of all allow lists - Patterns []string // All patterns (ANDed) - SetByPolicy string // Policy ID that first set SP constraints +// AccumulatedAgentConstraints tracks accumulated agent constraints across policies. +type AccumulatedAgentConstraints struct { + AllowList []string // Intersection of all allow lists + Patterns []string // All patterns (ANDed) + EnvironmentConstraints []string // Allowed environments + SetByPolicy string // Policy ID that first set agent constraints +} + +// MergeAgentConstraints merges agent constraints from a policy decision. +// Allow lists are intersected; patterns and environment constraints are appended (AND). +func (c *ConstraintContext) MergeAgentConstraints(incoming *AccumulatedAgentConstraints, policyID string) error { + if incoming == nil { + return nil + } + if len(incoming.AllowList) == 0 && len(incoming.Patterns) == 0 && len(incoming.EnvironmentConstraints) == 0 { + return nil + } + + if c.agentConstraints == nil { + c.agentConstraints = &AccumulatedAgentConstraints{ + AllowList: append([]string(nil), incoming.AllowList...), + Patterns: append([]string(nil), incoming.Patterns...), + EnvironmentConstraints: append([]string(nil), incoming.EnvironmentConstraints...), + SetByPolicy: policyID, + } + return nil + } + + if len(incoming.AllowList) > 0 && len(c.agentConstraints.AllowList) > 0 { + intersected := intersectStringSlices(c.agentConstraints.AllowList, incoming.AllowList) + if len(intersected) == 0 { + return fmt.Errorf("agent allow list intersection is empty: "+ + "policy '%s' allows %v but existing constraints from policy '%s' allow %v", + policyID, incoming.AllowList, c.agentConstraints.SetByPolicy, c.agentConstraints.AllowList) + } + c.agentConstraints.AllowList = intersected + } else if len(incoming.AllowList) > 0 { + c.agentConstraints.AllowList = incoming.AllowList + } + + c.agentConstraints.Patterns = append(c.agentConstraints.Patterns, incoming.Patterns...) + + if len(incoming.EnvironmentConstraints) > 0 && len(c.agentConstraints.EnvironmentConstraints) > 0 { + intersected := intersectStringSlices(c.agentConstraints.EnvironmentConstraints, incoming.EnvironmentConstraints) + if len(intersected) == 0 { + return fmt.Errorf("agent environment constraint intersection is empty: "+ + "policy '%s' allows %v but existing constraints from policy '%s' allow %v", + policyID, incoming.EnvironmentConstraints, c.agentConstraints.SetByPolicy, c.agentConstraints.EnvironmentConstraints) + } + c.agentConstraints.EnvironmentConstraints = intersected + } else if len(incoming.EnvironmentConstraints) > 0 { + c.agentConstraints.EnvironmentConstraints = incoming.EnvironmentConstraints + } + + return nil +} + +// ValidateAgent checks an agent name against availableAgents and the +// accumulated agent constraints. The availableAgents check is independent of +// (and enforced before) any policy allow-list/pattern below: a decision must +// stay within what was offered, regardless of what a policy further +// restricts it to. A nil/empty availableAgents means the check is skipped. +func (c *ConstraintContext) ValidateAgent(agentName string, availableAgents []string) error { + if agentName == "" { + return nil + } + + if len(availableAgents) > 0 && !slices.Contains(availableAgents, agentName) { + return fmt.Errorf("agent '%s' is not in the available agents list %v", agentName, availableAgents) + } + + if c.agentConstraints == nil { + return nil + } + + if len(c.agentConstraints.AllowList) > 0 { + if !slices.Contains(c.agentConstraints.AllowList, agentName) { + return fmt.Errorf("agent '%s' is not in the allowed list %v (constrained by policy '%s')", + agentName, c.agentConstraints.AllowList, c.agentConstraints.SetByPolicy) + } + } + + for _, pattern := range c.agentConstraints.Patterns { + matched, err := regexp.MatchString(pattern, agentName) + if err != nil { + return fmt.Errorf("invalid agent pattern '%s': %v", pattern, err) + } + if !matched { + return fmt.Errorf("agent '%s' does not match required pattern '%s'", agentName, pattern) + } + } + + return nil +} + +// ValidateAgentEnvironment checks an agent's environment against constraints. +func (c *ConstraintContext) ValidateAgentEnvironment(environment string) error { + if c.agentConstraints == nil || len(c.agentConstraints.EnvironmentConstraints) == 0 { + return nil + } + + if !slices.Contains(c.agentConstraints.EnvironmentConstraints, environment) { + return fmt.Errorf("agent environment '%s' not in allowed environments %v", + environment, c.agentConstraints.EnvironmentConstraints) + } + return nil +} + +// GetAgentConstraintsMap returns agent constraints for inclusion in OPA input. +func (c *ConstraintContext) GetAgentConstraintsMap() map[string]any { + if c.agentConstraints == nil { + return nil + } + result := make(map[string]any) + if len(c.agentConstraints.AllowList) > 0 { + result["allow_list"] = c.agentConstraints.AllowList + } + if len(c.agentConstraints.Patterns) > 0 { + result["patterns"] = c.agentConstraints.Patterns + } + if len(c.agentConstraints.EnvironmentConstraints) > 0 { + result["environment_constraints"] = c.agentConstraints.EnvironmentConstraints + } + return result } // ConstraintConflictError is returned by MergeConstraints when a lower-priority @@ -125,77 +243,6 @@ func (c *ConstraintContext) validatePatchRecursive( } } -// MergeSPConstraints merges service provider constraints from a policy decision. -// If sp is nil or has neither allow list nor patterns, it is a no-op. -// Allow lists are intersected once; all patterns are appended (ANDed). -func (c *ConstraintContext) MergeSPConstraints(sp *opa.ServiceProviderConstraints, policyID string) error { - if sp == nil { - return nil - } - allowList := sp.AllowList - patterns := sp.Patterns - if len(allowList) == 0 && len(patterns) == 0 { - return nil - } - if c.serviceProviderConstraints == nil { - c.serviceProviderConstraints = &AccumulatedSPConstraints{ - AllowList: allowList, - Patterns: append([]string(nil), patterns...), - SetByPolicy: policyID, - } - return nil - } - - // Intersect allow lists if both exist - if len(allowList) > 0 && len(c.serviceProviderConstraints.AllowList) > 0 { - intersected := intersectStringSlices(c.serviceProviderConstraints.AllowList, allowList) - if len(intersected) == 0 { - return fmt.Errorf("service provider allow list intersection is empty: "+ - "policy '%s' allows %v but existing constraints from policy '%s' allow %v", - policyID, allowList, c.serviceProviderConstraints.SetByPolicy, c.serviceProviderConstraints.AllowList) - } - c.serviceProviderConstraints.AllowList = intersected - } else if len(allowList) > 0 { - c.serviceProviderConstraints.AllowList = allowList - } - - // AND patterns - c.serviceProviderConstraints.Patterns = append(c.serviceProviderConstraints.Patterns, patterns...) - - return nil -} - -// ValidateServiceProvider checks a provider against accumulated SP constraints -func (c *ConstraintContext) ValidateServiceProvider(provider string) error { - if c.serviceProviderConstraints == nil || provider == "" { - return nil - } - - sp := c.serviceProviderConstraints - - // Check allow list - if len(sp.AllowList) > 0 { - found := slices.Contains(sp.AllowList, provider) - if !found { - return fmt.Errorf("provider '%s' is not in the allowed list %v (constrained by policy '%s')", - provider, sp.AllowList, sp.SetByPolicy) - } - } - - // Check patterns - for _, pattern := range sp.Patterns { - matched, err := regexp.MatchString(pattern, provider) - if err != nil { - return fmt.Errorf("invalid service provider pattern '%s': %v", pattern, err) - } - if !matched { - return fmt.Errorf("provider '%s' does not match required pattern '%s'", provider, pattern) - } - } - - return nil -} - // GetConstraintsMap returns the accumulated constraints for inclusion in OPA input func (c *ConstraintContext) GetConstraintsMap() map[string]any { if len(c.constrainedFieldsByFieldPath) == 0 { @@ -208,22 +255,6 @@ func (c *ConstraintContext) GetConstraintsMap() map[string]any { return result } -// GetSPConstraintsMap returns SP constraints for inclusion in OPA input -func (c *ConstraintContext) GetSPConstraintsMap() map[string]any { - if c.serviceProviderConstraints == nil { - return nil - } - result := make(map[string]any) - if len(c.serviceProviderConstraints.AllowList) > 0 { - result["allow_list"] = c.serviceProviderConstraints.AllowList - } - if len(c.serviceProviderConstraints.Patterns) > 0 { - // Combine patterns into a single regex with AND semantics - result["patterns"] = c.serviceProviderConstraints.Patterns - } - return result -} - // mergeSchemaKeywords merges JSON Schema keywords, enforcing tightening-only. func mergeSchemaKeywords(existing, new map[string]any, fieldPath, existingPolicyID string) (map[string]any, error) { merged := deepCopySchemaMap(existing) diff --git a/internal/policy/service/constraints_test.go b/internal/policy/service/constraints_test.go index 4d7d3ae..e9be467 100644 --- a/internal/policy/service/constraints_test.go +++ b/internal/policy/service/constraints_test.go @@ -1,7 +1,6 @@ package service import ( - "github.com/dcm-project/control-plane/internal/policy/opa" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -367,85 +366,6 @@ var _ = Describe("ConstraintContext", func() { }) }) - Describe("MergeSPConstraints", func() { - It("stores first SP constraints", func() { - err := constraintCtx.MergeSPConstraints(&opa.ServiceProviderConstraints{AllowList: []string{"aws", "gcp"}}, "policy-1") - Expect(err).NotTo(HaveOccurred()) - - spConstraints := constraintCtx.GetSPConstraintsMap() - Expect(spConstraints).To(HaveKey("allow_list")) - }) - - It("intersects allow lists", func() { - err := constraintCtx.MergeSPConstraints(&opa.ServiceProviderConstraints{AllowList: []string{"aws", "gcp", "azure"}}, "policy-1") - Expect(err).NotTo(HaveOccurred()) - - err = constraintCtx.MergeSPConstraints(&opa.ServiceProviderConstraints{AllowList: []string{"aws", "azure"}}, "policy-2") - Expect(err).NotTo(HaveOccurred()) - - spConstraints := constraintCtx.GetSPConstraintsMap() - allowList := spConstraints["allow_list"].([]string) - Expect(allowList).To(ConsistOf("aws", "azure")) - }) - - It("rejects empty allow list intersection", func() { - err := constraintCtx.MergeSPConstraints(&opa.ServiceProviderConstraints{AllowList: []string{"aws"}}, "policy-1") - Expect(err).NotTo(HaveOccurred()) - - err = constraintCtx.MergeSPConstraints(&opa.ServiceProviderConstraints{AllowList: []string{"gcp"}}, "policy-2") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("empty")) - }) - - It("accumulates patterns", func() { - err := constraintCtx.MergeSPConstraints(&opa.ServiceProviderConstraints{Patterns: []string{"^aws"}}, "policy-1") - Expect(err).NotTo(HaveOccurred()) - - err = constraintCtx.MergeSPConstraints(&opa.ServiceProviderConstraints{Patterns: []string{".*-prod$"}}, "policy-2") - Expect(err).NotTo(HaveOccurred()) - - spConstraints := constraintCtx.GetSPConstraintsMap() - patterns := spConstraints["patterns"].([]string) - Expect(patterns).To(ConsistOf("^aws", ".*-prod$")) - }) - }) - - Describe("ValidateServiceProvider", func() { - It("allows provider in allow list", func() { - err := constraintCtx.MergeSPConstraints(&opa.ServiceProviderConstraints{AllowList: []string{"aws", "gcp"}}, "policy-1") - Expect(err).NotTo(HaveOccurred()) - - err = constraintCtx.ValidateServiceProvider("aws") - Expect(err).NotTo(HaveOccurred()) - }) - - It("rejects provider not in allow list", func() { - err := constraintCtx.MergeSPConstraints(&opa.ServiceProviderConstraints{AllowList: []string{"aws", "gcp"}}, "policy-1") - Expect(err).NotTo(HaveOccurred()) - - err = constraintCtx.ValidateServiceProvider("azure") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("not in the allowed list")) - }) - - It("validates provider against pattern", func() { - err := constraintCtx.MergeSPConstraints(&opa.ServiceProviderConstraints{Patterns: []string{"^aws"}}, "policy-1") - Expect(err).NotTo(HaveOccurred()) - - err = constraintCtx.ValidateServiceProvider("aws-prod") - Expect(err).NotTo(HaveOccurred()) - - err = constraintCtx.ValidateServiceProvider("gcp-prod") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("does not match")) - }) - - It("allows empty provider without constraints", func() { - err := constraintCtx.ValidateServiceProvider("") - Expect(err).NotTo(HaveOccurred()) - }) - }) - Describe("GetConstraintsMap", func() { It("returns nil when no constraints exist", func() { Expect(constraintCtx.GetConstraintsMap()).To(BeNil()) diff --git a/internal/policy/service/errors.go b/internal/policy/service/errors.go index f48a8cf..a5a7d0c 100644 --- a/internal/policy/service/errors.go +++ b/internal/policy/service/errors.go @@ -128,6 +128,31 @@ func NewFailedPreconditionError(message, detail string) *ServiceError { } } +// NewNoCapableAgentError signals that agents were available but none serve +// the requested service type (406, same bucket as a policy rejection). +// remainingCount is the post-exclusion count, so the message doesn't imply +// "no agent anywhere supports this" when capable agents were excluded. +func NewNoCapableAgentError(serviceType string, remainingCount int) *ServiceError { + return &ServiceError{ + Type: ErrorTypeRejected, + Message: fmt.Sprintf("No available agent supports service type '%s'", serviceType), + Detail: fmt.Sprintf( + "service_type %q has no capable agent among the %d agent(s) remaining after exclusion filtering", + serviceType, remainingCount, + ), + } +} + +// NewAllAgentsExcludedError signals that exclude_agents removed every +// available agent (406, same bucket as a policy rejection). +func NewAllAgentsExcludedError(excludedCount int) *ServiceError { + return &ServiceError{ + Type: ErrorTypeRejected, + Message: "No available agent remains after exclusions", + Detail: fmt.Sprintf("all %d available agent(s) were excluded from consideration", excludedCount), + } +} + // NewPolicyRejectedError creates a new policy rejected error (406 Not Acceptable) func NewPolicyRejectedError(policyID, reason string) *ServiceError { return &ServiceError{ @@ -171,11 +196,11 @@ func NewConstraintConflictError(policyID, fieldPath, existingPolicyID, reason st } } -// NewServiceProviderConstraintError creates a new SP constraint error (409 Conflict) -func NewServiceProviderConstraintError(policyID, detail string) *ServiceError { +// NewAgentConstraintError creates an error when a selected agent violates constraints (409 Conflict) +func NewAgentConstraintError(policyID, detail string) *ServiceError { return &ServiceError{ Type: ErrorTypePolicyConflict, - Message: fmt.Sprintf("Policy '%s' selected a service provider that violates constraints", policyID), + Message: fmt.Sprintf("Policy '%s' selected an agent that violates constraints", policyID), Detail: detail, } } diff --git a/internal/policy/service/evaluation.go b/internal/policy/service/evaluation.go index 92bc61c..0bf848e 100644 --- a/internal/policy/service/evaluation.go +++ b/internal/policy/service/evaluation.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "slices" + "strings" "github.com/brunoga/deep/v4" "github.com/dcm-project/control-plane/internal/policy/logging" @@ -12,7 +14,6 @@ import ( "github.com/dcm-project/control-plane/internal/policy/store/model" ) -// EvaluationStatus represents the status of the evaluation type EvaluationStatus string const ( @@ -20,31 +21,37 @@ const ( EvaluationStatusModified EvaluationStatus = "MODIFIED" ) -// EvaluationService defines the interface for policy evaluation type EvaluationService interface { EvaluateRequest(ctx context.Context, req *EvaluationRequest) (*EvaluationResponse, error) } -// EvaluationRequest represents a request for policy evaluation +// AgentInfo is the subset of agent metadata policies need. Cost is "" when +// the agent didn't report one. +type AgentInfo struct { + Name string + Environment string + ServiceTypes []string + Cost string +} + type EvaluationRequest struct { ServiceInstance map[string]any RequestLabels map[string]string + AvailableAgents []AgentInfo + ExcludeAgents []string } -// EvaluationResponse represents the response from policy evaluation type EvaluationResponse struct { EvaluatedServiceInstance map[string]any - SelectedProvider string + SelectedAgent string Status EvaluationStatus } -// evaluationService implements EvaluationService type evaluationService struct { policyStore store.Policy engine opa.Engine } -// NewEvaluationService creates a new evaluation service func NewEvaluationService(policyStore store.Policy, engine opa.Engine) EvaluationService { return &evaluationService{ policyStore: policyStore, @@ -52,24 +59,48 @@ func NewEvaluationService(policyStore store.Policy, engine opa.Engine) Evaluatio } } -// EvaluateRequest evaluates a service instance request against all applicable policies func (s *evaluationService) EvaluateRequest(ctx context.Context, req *EvaluationRequest) (*EvaluationResponse, error) { log := logging.FromContext(ctx) log.Debug("Starting policy evaluation", "label_count", len(req.RequestLabels)) - // Initialize the current service instance spec (we'll modify this as we evaluate policies) currentSpec, err := deep.Copy(req.ServiceInstance) if err != nil { return nil, NewInternalError("Failed to make a deep copy of the service instance spec", err.Error(), err) } - // Initialize constraint context constraintCtx := NewConstraintContext() - // Track selected provider across policies (starts unknown) - selectedProvider := "" + totalAgents := len(req.AvailableAgents) + availableAgents := filterExcluded(req.AvailableAgents, req.ExcludeAgents) + + // ValidateAgent (constraints.go) skips its membership check on an empty + // available_agents list, so reject explicitly instead of letting a + // policy pick an unvalidated agent. totalAgents == 0 (no agent client + // configured) is left alone - that fail-open is intentional. + if totalAgents > 0 && len(availableAgents) == 0 { + log.Warn("All available agents were excluded", "excluded_count", len(req.ExcludeAgents)) + return nil, NewAllAgentsExcludedError(len(req.ExcludeAgents)) + } + + // Filtered here once rather than trusting every Rego policy to check + // capability itself, so an incapable agent doesn't surface later as an + // SP-side provisioning failure. + serviceType, hasServiceType, err := resourceServiceType(req.ServiceInstance) + if err != nil { + return nil, NewInvalidArgumentError("Invalid service_type in resource spec", err.Error()) + } + if hasServiceType { + capableAgents := filterByServiceType(availableAgents, serviceType) + if len(availableAgents) > 0 && len(capableAgents) == 0 { + log.Warn("No available agent supports the requested service type", + "service_type", serviceType, "agents_after_exclude", len(availableAgents)) + return nil, NewNoCapableAgentError(serviceType, len(availableAgents)) + } + availableAgents = capableAgents + } + + selectedAgent := "" - // Paginate over all enabled policies, ordered by policy_type ASC, priority ASC var pageToken *string policiesEvaluated := 0 policiesSkipped := 0 @@ -86,9 +117,7 @@ func (s *evaluationService) EvaluateRequest(ctx context.Context, req *Evaluation return nil, NewInternalError("Failed to retrieve policies", err.Error(), err) } - // Evaluate each policy on this page sequentially for _, policy := range policyListResult.Policies { - // Filter by label selector if !MatchesLabelSelector(policy.LabelSelector, req.RequestLabels) { policiesSkipped++ continue @@ -96,7 +125,7 @@ func (s *evaluationService) EvaluateRequest(ctx context.Context, req *Evaluation log.Debug("Evaluating policy", "policy_id", policy.ID, "policy_type", policy.PolicyType, "priority", policy.Priority) - currentSpec, selectedProvider, err = s.evaluatePolicy(ctx, &policy, currentSpec, selectedProvider, constraintCtx) + currentSpec, selectedAgent, err = s.evaluatePolicy(ctx, &policy, currentSpec, selectedAgent, availableAgents, req.ExcludeAgents, constraintCtx) if err != nil { log.Warn("Policy evaluation failed", "policy_id", policy.ID, "error", err) return nil, err @@ -110,7 +139,6 @@ func (s *evaluationService) EvaluateRequest(ctx context.Context, req *Evaluation pageToken = &policyListResult.NextPageToken } - // Determine status status := EvaluationStatusApproved if !deep.Equal(req.ServiceInstance, currentSpec) { status = EvaluationStatusModified @@ -120,12 +148,12 @@ func (s *evaluationService) EvaluateRequest(ctx context.Context, req *Evaluation "status", status, "policies_evaluated", policiesEvaluated, "policies_skipped", policiesSkipped, - "selected_provider", selectedProvider, + "selected_agent", selectedAgent, ) return &EvaluationResponse{ EvaluatedServiceInstance: currentSpec, - SelectedProvider: selectedProvider, + SelectedAgent: selectedAgent, Status: status, }, nil } @@ -134,23 +162,51 @@ func (s *evaluationService) evaluatePolicy( ctx context.Context, policy *model.Policy, currentSpec map[string]any, - selectedProvider string, + selectedAgent string, + availableAgents []AgentInfo, + excludeAgents []string, constraintCtx *ConstraintContext, ) (map[string]any, string, error) { log := logging.FromContext(ctx) - // 1. Build OPA input with constraints and SP constraints opaInput := map[string]any{ - "spec": currentSpec, - "provider": selectedProvider, + "spec": currentSpec, + } + if len(availableAgents) > 0 { + // Structured objects, not bare name strings, so Rego can reason + // about an agent's environment/capability/cost too. + agentsForOPA := make([]map[string]any, len(availableAgents)) + for i, a := range availableAgents { + serviceTypes := a.ServiceTypes + if serviceTypes == nil { + serviceTypes = []string{} // avoid Rego null + } + agentsForOPA[i] = map[string]any{ + "name": a.Name, + "environment": a.Environment, + "service_types": serviceTypes, + "cost": a.Cost, + } + } + opaInput["available_agents"] = agentsForOPA + } + // available_agents is already pre-filtered by filterExcluded, so + // policies can't see which agents were excluded from it (F38). Passing + // exclude_agents too lets Rego implement fallback logic that reasons + // about exclusions themselves (e.g. preferring an excluded agent's + // region peer), instead of only seeing the post-filter result. + if len(excludeAgents) > 0 { + opaInput["exclude_agents"] = excludeAgents + } + if selectedAgent != "" { + opaInput["agent"] = selectedAgent } if constraints := constraintCtx.GetConstraintsMap(); constraints != nil { opaInput["constraints"] = constraints } - if spConstraints := constraintCtx.GetSPConstraintsMap(); spConstraints != nil { - opaInput["service_provider_constraints"] = spConstraints + if agentConstraints := constraintCtx.GetAgentConstraintsMap(); agentConstraints != nil { + opaInput["agent_constraints"] = agentConstraints } - // 2. Evaluate the policy using the embedded engine evalResult, err := s.engine.EvaluatePolicy(ctx, policy.ID, opaInput) if err != nil { return nil, "", NewInternalError( @@ -160,22 +216,18 @@ func (s *evaluationService) evaluatePolicy( ) } - // Skip if policy is undefined if !evalResult.Defined { log.Debug("Policy returned undefined result, skipping", "policy_id", policy.ID) - return currentSpec, selectedProvider, nil + return currentSpec, selectedAgent, nil } - // Parse the policy decision decision := opa.ParsePolicyDecision(evalResult.Result) - // 3. Check for rejection if decision.Rejected { log.Info("Policy rejected request", "policy_id", policy.ID, "reason", decision.RejectionReason) return nil, "", NewPolicyRejectedError(policy.ID, decision.RejectionReason) } - // 4. Validate and merge constraints — new constraints must not loosen existing ones if decision.Constraints != nil { if err := constraintCtx.MergeConstraints(decision.Constraints, policy.ID); err != nil { var conflictErr *ConstraintConflictError @@ -188,19 +240,23 @@ func (s *evaluationService) evaluatePolicy( } } - // 5. Merge service provider constraints - if err := constraintCtx.MergeSPConstraints(decision.ServiceProviderConstraints, policy.ID); err != nil { - return nil, "", NewServiceProviderConstraintError(policy.ID, err.Error()) + if decision.AgentConstraints != nil { + ac := &AccumulatedAgentConstraints{ + AllowList: decision.AgentConstraints.AllowList, + Patterns: decision.AgentConstraints.Patterns, + EnvironmentConstraints: decision.AgentConstraints.EnvironmentConstraints, + } + if err := constraintCtx.MergeAgentConstraints(ac, policy.ID); err != nil { + return nil, "", NewAgentConstraintError(policy.ID, err.Error()) + } } - // 6. Validate patch against accumulated constraints if decision.Patch != nil { violations := constraintCtx.ValidatePatch(decision.Patch) if len(violations) > 0 { return nil, "", NewConstraintViolationError(policy.ID, violations) } - // 7. Apply patch — deep merge into currentSpec (RFC 7396 JSON Merge Patch semantics) currentSpec, err = mergePatch(currentSpec, decision.Patch) if err != nil { return nil, "", NewInternalError("Failed to merge patch into current spec", err.Error(), err) @@ -208,21 +264,25 @@ func (s *evaluationService) evaluatePolicy( log.Debug("Policy patch applied", "policy_id", policy.ID) } - // 8. Validate service provider against SP constraints - if decision.SelectedProvider != "" { - if err := constraintCtx.ValidateServiceProvider(decision.SelectedProvider); err != nil { - return nil, "", NewServiceProviderConstraintError(policy.ID, err.Error()) + if decision.SelectedAgent != "" { + if err := constraintCtx.ValidateAgent(decision.SelectedAgent, agentNames(availableAgents)); err != nil { + return nil, "", NewAgentConstraintError(policy.ID, err.Error()) + } + // A lookup miss here only happens when availableAgents was empty + // (ValidateAgent didn't enforce membership either), so there's + // nothing to validate the environment against. + if env, ok := agentEnvironment(availableAgents, decision.SelectedAgent); ok { + if err := constraintCtx.ValidateAgentEnvironment(env); err != nil { + return nil, "", NewAgentConstraintError(policy.ID, err.Error()) + } } - log.Debug("Policy selected provider", "policy_id", policy.ID, "provider", decision.SelectedProvider) - selectedProvider = decision.SelectedProvider + log.Debug("Policy selected agent", "policy_id", policy.ID, "agent", decision.SelectedAgent) + selectedAgent = decision.SelectedAgent } - return currentSpec, selectedProvider, nil + return currentSpec, selectedAgent, nil } -// mergePatch performs a recursive JSON Merge Patch (RFC 7396) of patch into base. -// Fields in patch override fields in base. Null values in patch remove fields from base. -// Fields not mentioned in patch are preserved from base. func mergePatch(base, patch map[string]any) (map[string]any, error) { result, err := deep.Copy(base) if err != nil { @@ -231,7 +291,6 @@ func mergePatch(base, patch map[string]any) (map[string]any, error) { for key, patchValue := range patch { if patchValue == nil { - // null means remove the field delete(result, key) continue } @@ -241,13 +300,11 @@ func mergePatch(base, patch map[string]any) (map[string]any, error) { baseMap, baseIsMap := baseValue.(map[string]any) if patchIsMap && baseExists && baseIsMap { - // Both are maps — recurse result[key], err = mergePatch(baseMap, patchMap) if err != nil { return nil, err } } else { - // Patch value overrides base result[key] = patchValue } } @@ -255,7 +312,79 @@ func mergePatch(base, patch map[string]any) (map[string]any, error) { return result, nil } -// boolPtr returns a pointer to a bool value +// resourceServiceType extracts "service_type" from the resource spec. ok is +// false when absent (skip capability filtering). A present-but-malformed +// value (non-string/empty) is a validation error rather than a skip, matching +// internal/sp's CreateInstance check on the same field. +func resourceServiceType(serviceInstance map[string]any) (serviceType string, ok bool, err error) { + v, present := serviceInstance["service_type"] + if !present { + return "", false, nil + } + s, isString := v.(string) + if !isString { + return "", false, fmt.Errorf("spec.service_type must be a string, got %T", v) + } + if strings.TrimSpace(s) == "" { + return "", false, errors.New("spec.service_type must not be empty") + } + return s, true, nil +} + +// filterByServiceType keeps only agents whose ServiceTypes includes +// serviceType. An agent with no ServiceTypes never matches - registration +// requires a non-empty list, so empty means "not a wildcard". +func filterByServiceType(available []AgentInfo, serviceType string) []AgentInfo { + var filtered []AgentInfo + for _, a := range available { + if slices.Contains(a.ServiceTypes, serviceType) { + filtered = append(filtered, a) + } + } + return filtered +} + +func filterExcluded(available []AgentInfo, excluded []string) []AgentInfo { + if len(excluded) == 0 { + return available + } + excludeSet := make(map[string]bool, len(excluded)) + for _, e := range excluded { + excludeSet[e] = true + } + var filtered []AgentInfo + for _, a := range available { + if !excludeSet[a.Name] { + filtered = append(filtered, a) + } + } + return filtered +} + +// agentNames extracts just the Name field, for callers (ValidateAgent) that +// only need membership-by-name and shouldn't otherwise depend on AgentInfo. +func agentNames(agents []AgentInfo) []string { + if len(agents) == 0 { + return nil + } + names := make([]string, len(agents)) + for i, a := range agents { + names[i] = a.Name + } + return names +} + +// agentEnvironment looks up the Environment of the AgentInfo with the given +// name. ok is false if no match is found (including when agents is empty). +func agentEnvironment(agents []AgentInfo, name string) (env string, ok bool) { + for _, a := range agents { + if a.Name == name { + return a.Environment, true + } + } + return "", false +} + func boolPtr(b bool) *bool { return &b } diff --git a/internal/policy/service/evaluation_test.go b/internal/policy/service/evaluation_test.go index 287db33..fc9b585 100644 --- a/internal/policy/service/evaluation_test.go +++ b/internal/policy/service/evaluation_test.go @@ -108,7 +108,7 @@ var _ = Describe("EvaluationService", func() { Expect(err).NotTo(HaveOccurred()) Expect(response.Status).To(Equal(EvaluationStatusApproved)) Expect(response.EvaluatedServiceInstance).To(Equal(map[string]any{})) - Expect(response.SelectedProvider).To(Equal("")) + Expect(response.SelectedAgent).To(Equal("")) }) }) @@ -153,12 +153,12 @@ var _ = Describe("EvaluationService", func() { "patch": map[string]any{ "region": "us-east-1", }, - "selected_provider": "aws", + "selected_agent": "aws-agent", }, } }) - It("returns modified with updated spec", func() { + It("returns modified with updated spec and selected agent", func() { response, err := service.EvaluateRequest(ctx, baseRequest) Expect(err).NotTo(HaveOccurred()) @@ -166,7 +166,7 @@ var _ = Describe("EvaluationService", func() { Expect(response.EvaluatedServiceInstance).To(Equal(map[string]any{ "region": "us-east-1", })) - Expect(response.SelectedProvider).To(Equal("aws")) + Expect(response.SelectedAgent).To(Equal("aws-agent")) }) }) @@ -595,7 +595,7 @@ var _ = Describe("EvaluationService", func() { }) }) - Context("when service provider constraints are enforced", func() { + Context("when agent constraints are enforced via allow_list", func() { BeforeEach(func() { mockStore.policies = []model.Policy{ { @@ -612,28 +612,26 @@ var _ = Describe("EvaluationService", func() { }, } - // First policy sets SP constraint allow list mockOPA.evaluations["policy-1"] = &opa.EvaluationResult{ Defined: true, Result: map[string]any{ "rejected": false, - "service_provider_constraints": map[string]any{ - "allow_list": []any{"aws", "gcp"}, + "agent_constraints": map[string]any{ + "allow_list": []any{"aws-agent", "gcp-agent"}, }, }, } - // Second policy selects a provider not in allow list mockOPA.evaluations["policy-2"] = &opa.EvaluationResult{ Defined: true, Result: map[string]any{ - "rejected": false, - "selected_provider": "azure", + "rejected": false, + "selected_agent": "azure-agent", }, } }) - It("returns SP constraint error", func() { + It("returns agent constraint error", func() { _, err := service.EvaluateRequest(ctx, baseRequest) Expect(err).To(HaveOccurred()) @@ -641,12 +639,468 @@ var _ = Describe("EvaluationService", func() { Expect(ok).To(BeTrue()) Expect(serviceErr.Type).To(Equal(ErrorTypePolicyConflict)) Expect(serviceErr.Message).To(ContainSubstring("policy-2")) - Expect(serviceErr.Detail).To(ContainSubstring("not in the allowed list")) }) }) }) }) +var _ = Describe("EvaluateRequest with agents", func() { + var ( + ctx context.Context + mockStore *mockPolicyStore + mockOPA *mockEngine + svc EvaluationService + baseRequest *EvaluationRequest + ) + + BeforeEach(func() { + ctx = context.Background() + mockStore = &mockPolicyStore{policies: []model.Policy{}} + mockOPA = &mockEngine{evaluations: make(map[string]*opa.EvaluationResult)} + svc = NewEvaluationService(mockStore, mockOPA) + baseRequest = &EvaluationRequest{ + ServiceInstance: map[string]any{}, + RequestLabels: map[string]string{}, + } + }) + + It("passes available_agents to OPA input", func() { + baseRequest.AvailableAgents = agentInfosForVM("agent-a", "agent-b") + baseRequest.ServiceInstance["service_type"] = "vm" + + var capturedInput map[string]any + captureOPA := &mockEngineWithCapture{ + evaluations: make(map[string]*opa.EvaluationResult), + captureFunc: func(input map[string]any) { capturedInput = input }, + } + svc = NewEvaluationService(mockStore, captureOPA) + + mockStore.policies = []model.Policy{ + {ID: "p1", PolicyType: "routing", Priority: 1, Enabled: true}, + } + captureOPA.evaluations["p1"] = &opa.EvaluationResult{Defined: true, Result: map[string]any{}} + + _, err := svc.EvaluateRequest(ctx, baseRequest) + Expect(err).NotTo(HaveOccurred()) + Expect(capturedInput).To(HaveKey("available_agents")) + }) + + It("pre-filters exclude_agents before evaluation", func() { + baseRequest.AvailableAgents = agentInfosForVM("agent-a", "agent-b", "agent-c") + baseRequest.ExcludeAgents = []string{"agent-b"} + baseRequest.ServiceInstance["service_type"] = "vm" + + var capturedInput map[string]any + captureOPA := &mockEngineWithCapture{ + evaluations: make(map[string]*opa.EvaluationResult), + captureFunc: func(input map[string]any) { capturedInput = input }, + } + svc = NewEvaluationService(mockStore, captureOPA) + + mockStore.policies = []model.Policy{ + {ID: "p1", PolicyType: "routing", Priority: 1, Enabled: true}, + } + captureOPA.evaluations["p1"] = &opa.EvaluationResult{Defined: true, Result: map[string]any{}} + + _, err := svc.EvaluateRequest(ctx, baseRequest) + Expect(err).NotTo(HaveOccurred()) + + agents, ok := capturedInput["available_agents"].([]map[string]any) + Expect(ok).To(BeTrue()) + names := make([]string, 0, len(agents)) + for _, a := range agents { + names = append(names, a["name"].(string)) + } + Expect(names).NotTo(ContainElement("agent-b")) + }) + + It("includes exclude_agents in the OPA input for policy transparency", func() { + baseRequest.AvailableAgents = agentInfosForVM("agent-a", "agent-b", "agent-c") + baseRequest.ExcludeAgents = []string{"agent-b"} + baseRequest.ServiceInstance["service_type"] = "vm" + + var capturedInput map[string]any + captureOPA := &mockEngineWithCapture{ + evaluations: make(map[string]*opa.EvaluationResult), + captureFunc: func(input map[string]any) { capturedInput = input }, + } + svc = NewEvaluationService(mockStore, captureOPA) + + mockStore.policies = []model.Policy{ + {ID: "p1", PolicyType: "routing", Priority: 1, Enabled: true}, + } + captureOPA.evaluations["p1"] = &opa.EvaluationResult{Defined: true, Result: map[string]any{}} + + _, err := svc.EvaluateRequest(ctx, baseRequest) + Expect(err).NotTo(HaveOccurred()) + + excluded, ok := capturedInput["exclude_agents"].([]string) + Expect(ok).To(BeTrue()) + Expect(excluded).To(ConsistOf("agent-b")) + }) + + It("omits exclude_agents from the OPA input when nothing was excluded", func() { + baseRequest.AvailableAgents = agentInfosForVM("agent-a") + baseRequest.ServiceInstance["service_type"] = "vm" + + var capturedInput map[string]any + captureOPA := &mockEngineWithCapture{ + evaluations: make(map[string]*opa.EvaluationResult), + captureFunc: func(input map[string]any) { capturedInput = input }, + } + svc = NewEvaluationService(mockStore, captureOPA) + + mockStore.policies = []model.Policy{ + {ID: "p1", PolicyType: "routing", Priority: 1, Enabled: true}, + } + captureOPA.evaluations["p1"] = &opa.EvaluationResult{Defined: true, Result: map[string]any{}} + + _, err := svc.EvaluateRequest(ctx, baseRequest) + Expect(err).NotTo(HaveOccurred()) + Expect(capturedInput).NotTo(HaveKey("exclude_agents")) + }) + + It("returns selected_agent in response", func() { + mockStore.policies = []model.Policy{ + {ID: "agent-policy", PolicyType: "routing", Priority: 1, Enabled: true}, + } + mockOPA.evaluations["agent-policy"] = &opa.EvaluationResult{ + Defined: true, + Result: map[string]any{ + "selected_agent": "my-agent", + }, + } + + resp, err := svc.EvaluateRequest(ctx, baseRequest) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.SelectedAgent).To(Equal("my-agent")) + }) + + It("validates selected_agent against agent_constraints", func() { + mockStore.policies = []model.Policy{ + {ID: "constraint-policy", PolicyType: "constraint", Priority: 1, Enabled: true}, + {ID: "routing-policy", PolicyType: "routing", Priority: 2, Enabled: true}, + } + mockOPA.evaluations["constraint-policy"] = &opa.EvaluationResult{ + Defined: true, + Result: map[string]any{ + "agent_constraints": map[string]any{ + "allow_list": []any{"allowed-agent"}, + }, + }, + } + mockOPA.evaluations["routing-policy"] = &opa.EvaluationResult{ + Defined: true, + Result: map[string]any{ + "selected_agent": "denied-agent", + }, + } + + _, err := svc.EvaluateRequest(ctx, baseRequest) + Expect(err).To(HaveOccurred()) + }) + + // A misconfigured policy with no agent_constraints must still be + // rejected if it selects an agent outside available_agents. + It("rejects a selected_agent absent from AvailableAgents even when no agent_constraints policy ran", func() { + baseRequest.AvailableAgents = agentInfos("agent-a", "agent-b") + mockStore.policies = []model.Policy{ + {ID: "routing-policy", PolicyType: "routing", Priority: 1, Enabled: true}, + } + mockOPA.evaluations["routing-policy"] = &opa.EvaluationResult{ + Defined: true, + Result: map[string]any{ + "selected_agent": "rogue-agent", + }, + } + + _, err := svc.EvaluateRequest(ctx, baseRequest) + + Expect(err).To(HaveOccurred()) + serviceErr, ok := err.(*ServiceError) + Expect(ok).To(BeTrue()) + Expect(serviceErr.Detail).To(ContainSubstring("rogue-agent")) + }) + + It("accepts a selected_agent present in AvailableAgents when no agent_constraints policy ran", func() { + baseRequest.AvailableAgents = agentInfos("agent-a", "agent-b") + mockStore.policies = []model.Policy{ + {ID: "routing-policy", PolicyType: "routing", Priority: 1, Enabled: true}, + } + mockOPA.evaluations["routing-policy"] = &opa.EvaluationResult{ + Defined: true, + Result: map[string]any{ + "selected_agent": "agent-b", + }, + } + + resp, err := svc.EvaluateRequest(ctx, baseRequest) + + Expect(err).NotTo(HaveOccurred()) + Expect(resp.SelectedAgent).To(Equal("agent-b")) + }) +}) + +var _ = Describe("EvaluateRequest service-type capability filtering", func() { + var ( + ctx context.Context + mockStore *mockPolicyStore + mockOPA *mockEngine + svc EvaluationService + baseRequest *EvaluationRequest + ) + + BeforeEach(func() { + ctx = context.Background() + mockStore = &mockPolicyStore{policies: []model.Policy{ + {ID: "routing-policy", PolicyType: "routing", Priority: 1, Enabled: true}, + }} + mockOPA = &mockEngine{evaluations: map[string]*opa.EvaluationResult{ + "routing-policy": {Defined: true, Result: map[string]any{"selected_agent": "db-agent"}}, + }} + svc = NewEvaluationService(mockStore, mockOPA) + baseRequest = &EvaluationRequest{ + ServiceInstance: map[string]any{"service_type": "database"}, + RequestLabels: map[string]string{}, + AvailableAgents: []AgentInfo{ + {Name: "vm-agent", ServiceTypes: []string{"vm"}}, + {Name: "db-agent", ServiceTypes: []string{"database"}}, + }, + } + }) + + It("excludes agents that don't support the requested service type from evaluation", func() { + captured := map[string]any{} + captureOPA := &mockEngineWithCapture{ + evaluations: mockOPA.evaluations, + captureFunc: func(input map[string]any) { captured = input }, + } + svc = NewEvaluationService(mockStore, captureOPA) + + resp, err := svc.EvaluateRequest(ctx, baseRequest) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.SelectedAgent).To(Equal("db-agent")) + + agents, ok := captured["available_agents"].([]map[string]any) + Expect(ok).To(BeTrue()) + names := make([]string, 0, len(agents)) + for _, a := range agents { + names = append(names, a["name"].(string)) + } + Expect(names).To(ConsistOf("db-agent")) + }) + + It("exposes each agent's service_types in the OPA input", func() { + var captured map[string]any + captureOPA := &mockEngineWithCapture{ + evaluations: mockOPA.evaluations, + captureFunc: func(input map[string]any) { captured = input }, + } + svc = NewEvaluationService(mockStore, captureOPA) + + _, err := svc.EvaluateRequest(ctx, baseRequest) + Expect(err).NotTo(HaveOccurred()) + + agents, ok := captured["available_agents"].([]map[string]any) + Expect(ok).To(BeTrue()) + // Only db-agent survives the capability filter (vm-agent is + // dropped), so assert unconditionally rather than inside a + // name-matching loop that would pass vacuously if filtering + // silently returned zero agents. + Expect(agents).To(HaveLen(1)) + Expect(agents[0]["name"]).To(Equal("db-agent")) + Expect(agents[0]["service_types"]).To(Equal([]string{"database"})) + }) + + It("exposes an empty array (not null) for an agent with nil ServiceTypes when no capability filter runs", func() { + baseRequest.ServiceInstance = map[string]any{} + baseRequest.AvailableAgents = []AgentInfo{{Name: "no-types-agent"}} + mockOPA.evaluations["routing-policy"] = &opa.EvaluationResult{ + Defined: true, + Result: map[string]any{"selected_agent": "no-types-agent"}, + } + + var captured map[string]any + captureOPA := &mockEngineWithCapture{ + evaluations: mockOPA.evaluations, + captureFunc: func(input map[string]any) { captured = input }, + } + svc = NewEvaluationService(mockStore, captureOPA) + + _, err := svc.EvaluateRequest(ctx, baseRequest) + Expect(err).NotTo(HaveOccurred()) + + agents, ok := captured["available_agents"].([]map[string]any) + Expect(ok).To(BeTrue()) + Expect(agents).To(HaveLen(1)) + Expect(agents[0]["service_types"]).To(Equal([]string{})) + }) + + It("matches when an agent supports the requested type among several", func() { + baseRequest.AvailableAgents = []AgentInfo{ + {Name: "multi-agent", ServiceTypes: []string{"vm", "database", "storage"}}, + } + mockOPA.evaluations["routing-policy"] = &opa.EvaluationResult{ + Defined: true, + Result: map[string]any{"selected_agent": "multi-agent"}, + } + + resp, err := svc.EvaluateRequest(ctx, baseRequest) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.SelectedAgent).To(Equal("multi-agent")) + }) + + It("rejects with a clear error when excluding the only capable agent leaves none, even though other (incapable) agents remain", func() { + baseRequest.ExcludeAgents = []string{"db-agent"} + + _, err := svc.EvaluateRequest(ctx, baseRequest) + + Expect(err).To(HaveOccurred()) + serviceErr, ok := err.(*ServiceError) + Expect(ok).To(BeTrue()) + Expect(serviceErr.Type).To(Equal(ErrorTypeRejected)) + Expect(serviceErr.Detail).To(ContainSubstring("database")) + Expect(serviceErr.Detail).To(ContainSubstring("1 agent")) + }) + + It("rejects with a validation error when service_type is present but not a string", func() { + baseRequest.ServiceInstance["service_type"] = 42 + + _, err := svc.EvaluateRequest(ctx, baseRequest) + + Expect(err).To(HaveOccurred()) + serviceErr, ok := err.(*ServiceError) + Expect(ok).To(BeTrue()) + Expect(serviceErr.Type).To(Equal(ErrorTypeInvalidArgument)) + }) + + It("rejects with a validation error when service_type is an empty or whitespace-only string", func() { + baseRequest.ServiceInstance["service_type"] = " " + + _, err := svc.EvaluateRequest(ctx, baseRequest) + + Expect(err).To(HaveOccurred()) + serviceErr, ok := err.(*ServiceError) + Expect(ok).To(BeTrue()) + Expect(serviceErr.Type).To(Equal(ErrorTypeInvalidArgument)) + }) + + It("rejects with a clear error when no available agent supports the requested service type", func() { + baseRequest.ServiceInstance["service_type"] = "storage" + + _, err := svc.EvaluateRequest(ctx, baseRequest) + + Expect(err).To(HaveOccurred()) + serviceErr, ok := err.(*ServiceError) + Expect(ok).To(BeTrue()) + Expect(serviceErr.Type).To(Equal(ErrorTypeRejected)) + Expect(serviceErr.Message).To(ContainSubstring("storage")) + }) + + It("does not filter when the service instance has no service_type", func() { + baseRequest.ServiceInstance = map[string]any{} + + captured := map[string]any{} + captureOPA := &mockEngineWithCapture{ + evaluations: mockOPA.evaluations, + captureFunc: func(input map[string]any) { captured = input }, + } + svc = NewEvaluationService(mockStore, captureOPA) + + _, err := svc.EvaluateRequest(ctx, baseRequest) + Expect(err).NotTo(HaveOccurred()) + + agents, ok := captured["available_agents"].([]map[string]any) + Expect(ok).To(BeTrue()) + Expect(agents).To(HaveLen(2)) + }) + + It("does not filter agents that were already excluded via exclude_agents", func() { + // Both remaining candidates support "database", so exclusion alone + // (not capability) determines what's left. + baseRequest.AvailableAgents = []AgentInfo{ + {Name: "db-agent-1", ServiceTypes: []string{"database"}}, + {Name: "db-agent-2", ServiceTypes: []string{"database"}}, + } + baseRequest.ExcludeAgents = []string{"db-agent-1"} + mockOPA.evaluations["routing-policy"] = &opa.EvaluationResult{ + Defined: true, + Result: map[string]any{"selected_agent": "db-agent-2"}, + } + + resp, err := svc.EvaluateRequest(ctx, baseRequest) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.SelectedAgent).To(Equal("db-agent-2")) + }) +}) + +var _ = Describe("EvaluateRequest all-agents-excluded guard", func() { + var ( + ctx context.Context + mockStore *mockPolicyStore + mockOPA *mockEngine + svc EvaluationService + baseRequest *EvaluationRequest + ) + + BeforeEach(func() { + ctx = context.Background() + mockStore = &mockPolicyStore{policies: []model.Policy{}} + mockOPA = &mockEngine{evaluations: make(map[string]*opa.EvaluationResult)} + svc = NewEvaluationService(mockStore, mockOPA) + baseRequest = &EvaluationRequest{ + ServiceInstance: map[string]any{}, + RequestLabels: map[string]string{}, + AvailableAgents: agentInfos("only-agent"), + ExcludeAgents: []string{"only-agent"}, + } + }) + + It("rejects when exclusion removes every available agent, instead of falling through to Rego with none", func() { + _, err := svc.EvaluateRequest(ctx, baseRequest) + + Expect(err).To(HaveOccurred()) + serviceErr, ok := err.(*ServiceError) + Expect(ok).To(BeTrue()) + Expect(serviceErr.Type).To(Equal(ErrorTypeRejected)) + Expect(serviceErr.Detail).To(ContainSubstring("1")) + }) + + It("does not reject when there were no available agents to begin with (no agent client configured)", func() { + baseRequest.AvailableAgents = nil + baseRequest.ExcludeAgents = nil + mockStore.policies = []model.Policy{ + {ID: "p1", PolicyType: "routing", Priority: 1, Enabled: true}, + } + mockOPA.evaluations["p1"] = &opa.EvaluationResult{Defined: true, Result: map[string]any{}} + + _, err := svc.EvaluateRequest(ctx, baseRequest) + Expect(err).NotTo(HaveOccurred()) + }) +}) + +// agentInfos builds []AgentInfo from bare names (Environment left blank) +// for tests that only care about name-based membership/filtering. +func agentInfos(names ...string) []AgentInfo { + infos := make([]AgentInfo, len(names)) + for i, n := range names { + infos[i] = AgentInfo{Name: n} + } + return infos +} + +// agentInfosForVM builds []AgentInfo from bare names, each declaring "vm" +// as a supported service type, for tests that set +// ServiceInstance["service_type"] = "vm" but only care about name-based +// membership/filtering (not capability filtering itself). +func agentInfosForVM(names ...string) []AgentInfo { + infos := make([]AgentInfo, len(names)) + for i, n := range names { + infos[i] = AgentInfo{Name: n, ServiceTypes: []string{"vm"}} + } + return infos +} + // mockEngineWithCapture wraps mockEngine and captures inputs type mockEngineWithCapture struct { evaluations map[string]*opa.EvaluationResult diff --git a/internal/sp/api/provider/server.gen.go b/internal/sp/api/provider/server.gen.go deleted file mode 100644 index 1d397e9..0000000 --- a/internal/sp/api/provider/server.gen.go +++ /dev/null @@ -1,1354 +0,0 @@ -// Package provider provides primitives to interact with the openapi HTTP API. -// -// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.7.0 DO NOT EDIT. -package provider - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "time" - - "github.com/go-chi/chi/v5" - "github.com/oapi-codegen/runtime" -) - -const ( - BearerAuthScopes bearerAuthContextKey = "bearerAuth.Scopes" -) - -// Error RFC 7807 compliant error response -type Error struct { - // Detail Human-readable explanation specific to this occurrence - Detail *string `json:"detail,omitempty"` - - // Instance URI reference for this specific error occurrence - Instance *string `json:"instance,omitempty"` - - // Status HTTP status code - Status *int `json:"status,omitempty"` - - // Title Short human-readable summary of the problem - Title string `json:"title"` - - // Type URI reference identifying the error type - Type string `json:"type"` -} - -// Provider Full provider resource representation -type Provider struct { - // CreateTime Timestamp when the provider was first registered - CreateTime *time.Time `json:"create_time,omitempty"` - - // DisplayName Human-readable display name for the provider - DisplayName *string `json:"display_name,omitempty"` - - // Endpoint Full endpoint URL where the provider API is accessible - Endpoint string `json:"endpoint"` - - // HealthStatus Health status of the provider - HealthStatus *string `json:"health_status,omitempty"` - - // Id Unique identifier for the Service Provider - Id *string `json:"id,omitempty"` - - // Metadata Additional metadata about the provider - Metadata *ProviderMetadata `json:"metadata,omitempty"` - - // Name Unique name of the Service Provider - Name string `json:"name"` - - // Operations List of operations supported for this service type - Operations *[]string `json:"operations,omitempty"` - - // Path Resource path identifier - Path *string `json:"path,omitempty"` - - // SchemaVersion Schema version of the service type the SP supports - SchemaVersion string `json:"schema_version"` - - // ServiceType Type of service this provider offers - ServiceType string `json:"service_type"` - - // UpdateTime Timestamp when the provider was last updated - UpdateTime *time.Time `json:"update_time,omitempty"` -} - -// ProviderList Paginated list of providers -type ProviderList struct { - // NextPageToken Token for retrieving the next page of results - NextPageToken *string `json:"next_page_token,omitempty"` - Providers *[]Provider `json:"providers,omitempty"` -} - -// ProviderMetadata Additional metadata about the provider -type ProviderMetadata struct { - // RegionCode Geographic region code where the provider operates - RegionCode *string `json:"region_code,omitempty"` - - // Resources Resource capacity information - Resources *ResourceCapacity `json:"resources,omitempty"` - - // Status Current operational status of the provider - Status *string `json:"status,omitempty"` - - // Zone Availability zone or datacenter identifier - Zone *string `json:"zone,omitempty"` - AdditionalProperties map[string]interface{} `json:"-"` -} - -// ResourceCapacity Resource capacity information -type ResourceCapacity struct { - // TotalCpu Total CPU cores available - TotalCpu *int `json:"total_cpu,omitempty"` - - // TotalMemory Total memory available - TotalMemory *string `json:"total_memory,omitempty"` - - // TotalNode Total number of nodes - TotalNode *int `json:"total_node,omitempty"` - - // TotalStorage Total storage available - TotalStorage *string `json:"total_storage,omitempty"` -} - -// ProviderIdPath defines model for ProviderIdPath. -type ProviderIdPath = string - -// Forbidden RFC 7807 compliant error response -type Forbidden = Error - -// Unauthorized RFC 7807 compliant error response -type Unauthorized = Error - -// bearerAuthContextKey is the context key for bearerAuth security scheme -type bearerAuthContextKey string - -// ListProvidersParams defines parameters for ListProviders. -type ListProvidersParams struct { - // Type Filter providers by service type - Type *string `form:"type,omitempty" json:"type,omitempty"` - - // MaxPageSize Maximum number of results per page - MaxPageSize *int `form:"max_page_size,omitempty" json:"max_page_size,omitempty"` - - // PageToken Token for pagination - PageToken *string `form:"page_token,omitempty" json:"page_token,omitempty"` -} - -// CreateProviderParams defines parameters for CreateProvider. -type CreateProviderParams struct { - // Id Optional provider ID for idempotent registration - Id *string `form:"id,omitempty" json:"id,omitempty"` -} - -// CreateProviderJSONRequestBody defines body for CreateProvider for application/json ContentType. -type CreateProviderJSONRequestBody = Provider - -// ApplyProviderJSONRequestBody defines body for ApplyProvider for application/json ContentType. -type ApplyProviderJSONRequestBody = Provider - -// Getter for additional properties for ProviderMetadata. Returns the specified -// element and whether it was found -func (a ProviderMetadata) Get(fieldName string) (value interface{}, found bool) { - if a.AdditionalProperties != nil { - value, found = a.AdditionalProperties[fieldName] - } - return -} - -// Setter for additional properties for ProviderMetadata -func (a *ProviderMetadata) Set(fieldName string, value interface{}) { - if a.AdditionalProperties == nil { - a.AdditionalProperties = make(map[string]interface{}) - } - a.AdditionalProperties[fieldName] = value -} - -// Override default JSON handling for ProviderMetadata to handle AdditionalProperties -func (a *ProviderMetadata) UnmarshalJSON(b []byte) error { - object := make(map[string]json.RawMessage) - err := json.Unmarshal(b, &object) - if err != nil { - return err - } - - if raw, found := object["region_code"]; found { - err = json.Unmarshal(raw, &a.RegionCode) - if err != nil { - return fmt.Errorf("error reading 'region_code': %w", err) - } - delete(object, "region_code") - } - - if raw, found := object["resources"]; found { - err = json.Unmarshal(raw, &a.Resources) - if err != nil { - return fmt.Errorf("error reading 'resources': %w", err) - } - delete(object, "resources") - } - - if raw, found := object["status"]; found { - err = json.Unmarshal(raw, &a.Status) - if err != nil { - return fmt.Errorf("error reading 'status': %w", err) - } - delete(object, "status") - } - - if raw, found := object["zone"]; found { - err = json.Unmarshal(raw, &a.Zone) - if err != nil { - return fmt.Errorf("error reading 'zone': %w", err) - } - delete(object, "zone") - } - - if len(object) != 0 { - a.AdditionalProperties = make(map[string]interface{}) - for fieldName, fieldBuf := range object { - var fieldVal interface{} - err := json.Unmarshal(fieldBuf, &fieldVal) - if err != nil { - return fmt.Errorf("error unmarshaling field %s: %w", fieldName, err) - } - a.AdditionalProperties[fieldName] = fieldVal - } - } - return nil -} - -// Override default JSON handling for ProviderMetadata to handle AdditionalProperties -func (a ProviderMetadata) MarshalJSON() ([]byte, error) { - var err error - object := make(map[string]json.RawMessage) - - if a.RegionCode != nil { - object["region_code"], err = json.Marshal(a.RegionCode) - if err != nil { - return nil, fmt.Errorf("error marshaling 'region_code': %w", err) - } - } - - if a.Resources != nil { - object["resources"], err = json.Marshal(a.Resources) - if err != nil { - return nil, fmt.Errorf("error marshaling 'resources': %w", err) - } - } - - if a.Status != nil { - object["status"], err = json.Marshal(a.Status) - if err != nil { - return nil, fmt.Errorf("error marshaling 'status': %w", err) - } - } - - if a.Zone != nil { - object["zone"], err = json.Marshal(a.Zone) - if err != nil { - return nil, fmt.Errorf("error marshaling 'zone': %w", err) - } - } - - for fieldName, field := range a.AdditionalProperties { - object[fieldName], err = json.Marshal(field) - if err != nil { - return nil, fmt.Errorf("error marshaling '%s': %w", fieldName, err) - } - } - return json.Marshal(object) -} - -// ServerInterface represents all server handlers. -type ServerInterface interface { - // List all providers - // (GET /providers) - ListProviders(w http.ResponseWriter, r *http.Request, params ListProvidersParams) - // Register a Service Provider - // (POST /providers) - CreateProvider(w http.ResponseWriter, r *http.Request, params CreateProviderParams) - // Delete a service Provider - // (DELETE /providers/{providerId}) - DeleteProvider(w http.ResponseWriter, r *http.Request, providerId ProviderIdPath) - // Get a provider - // (GET /providers/{providerId}) - GetProvider(w http.ResponseWriter, r *http.Request, providerId ProviderIdPath) - // Update a Service Provider - // (PUT /providers/{providerId}) - ApplyProvider(w http.ResponseWriter, r *http.Request, providerId ProviderIdPath) -} - -// Unimplemented server implementation that returns http.StatusNotImplemented for each endpoint. - -type Unimplemented struct{} - -// List all providers -// (GET /providers) -func (_ Unimplemented) ListProviders(w http.ResponseWriter, r *http.Request, params ListProvidersParams) { - w.WriteHeader(http.StatusNotImplemented) -} - -// Register a Service Provider -// (POST /providers) -func (_ Unimplemented) CreateProvider(w http.ResponseWriter, r *http.Request, params CreateProviderParams) { - w.WriteHeader(http.StatusNotImplemented) -} - -// Delete a service Provider -// (DELETE /providers/{providerId}) -func (_ Unimplemented) DeleteProvider(w http.ResponseWriter, r *http.Request, providerId ProviderIdPath) { - w.WriteHeader(http.StatusNotImplemented) -} - -// Get a provider -// (GET /providers/{providerId}) -func (_ Unimplemented) GetProvider(w http.ResponseWriter, r *http.Request, providerId ProviderIdPath) { - w.WriteHeader(http.StatusNotImplemented) -} - -// Update a Service Provider -// (PUT /providers/{providerId}) -func (_ Unimplemented) ApplyProvider(w http.ResponseWriter, r *http.Request, providerId ProviderIdPath) { - w.WriteHeader(http.StatusNotImplemented) -} - -// ServerInterfaceWrapper converts contexts to parameters. -type ServerInterfaceWrapper struct { - Handler ServerInterface - HandlerMiddlewares []MiddlewareFunc - ErrorHandlerFunc func(w http.ResponseWriter, r *http.Request, err error) -} - -type MiddlewareFunc func(http.Handler) http.Handler - -// ListProviders operation middleware -func (siw *ServerInterfaceWrapper) ListProviders(w http.ResponseWriter, r *http.Request) { - - var err error - _ = err - - ctx := r.Context() - - ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) - - r = r.WithContext(ctx) - - // Parameter object where we will unmarshal all parameters from the context - var params ListProvidersParams - - // ------------- Optional query parameter "type" ------------- - - err = runtime.BindQueryParameterWithOptions("form", true, false, "type", r.URL.Query(), ¶ms.Type, runtime.BindQueryParameterOptions{Type: "string", Format: ""}) - if err != nil { - var requiredError *runtime.RequiredParameterError - if errors.As(err, &requiredError) { - siw.ErrorHandlerFunc(w, r, &RequiredParamError{ParamName: "type"}) - } else { - siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "type", Err: err}) - } - return - } - - // ------------- Optional query parameter "max_page_size" ------------- - - err = runtime.BindQueryParameterWithOptions("form", true, false, "max_page_size", r.URL.Query(), ¶ms.MaxPageSize, runtime.BindQueryParameterOptions{Type: "integer", Format: ""}) - if err != nil { - var requiredError *runtime.RequiredParameterError - if errors.As(err, &requiredError) { - siw.ErrorHandlerFunc(w, r, &RequiredParamError{ParamName: "max_page_size"}) - } else { - siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "max_page_size", Err: err}) - } - return - } - - // ------------- Optional query parameter "page_token" ------------- - - err = runtime.BindQueryParameterWithOptions("form", true, false, "page_token", r.URL.Query(), ¶ms.PageToken, runtime.BindQueryParameterOptions{Type: "string", Format: ""}) - if err != nil { - var requiredError *runtime.RequiredParameterError - if errors.As(err, &requiredError) { - siw.ErrorHandlerFunc(w, r, &RequiredParamError{ParamName: "page_token"}) - } else { - siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "page_token", Err: err}) - } - return - } - - handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - siw.Handler.ListProviders(w, r, params) - })) - - for _, middleware := range siw.HandlerMiddlewares { - handler = middleware(handler) - } - - handler.ServeHTTP(w, r) -} - -// CreateProvider operation middleware -func (siw *ServerInterfaceWrapper) CreateProvider(w http.ResponseWriter, r *http.Request) { - - var err error - _ = err - - ctx := r.Context() - - ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) - - r = r.WithContext(ctx) - - // Parameter object where we will unmarshal all parameters from the context - var params CreateProviderParams - - // ------------- Optional query parameter "id" ------------- - - err = runtime.BindQueryParameterWithOptions("form", true, false, "id", r.URL.Query(), ¶ms.Id, runtime.BindQueryParameterOptions{Type: "string", Format: ""}) - if err != nil { - var requiredError *runtime.RequiredParameterError - if errors.As(err, &requiredError) { - siw.ErrorHandlerFunc(w, r, &RequiredParamError{ParamName: "id"}) - } else { - siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) - } - return - } - - handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - siw.Handler.CreateProvider(w, r, params) - })) - - for _, middleware := range siw.HandlerMiddlewares { - handler = middleware(handler) - } - - handler.ServeHTTP(w, r) -} - -// DeleteProvider operation middleware -func (siw *ServerInterfaceWrapper) DeleteProvider(w http.ResponseWriter, r *http.Request) { - - var err error - _ = err - - // ------------- Path parameter "providerId" ------------- - var providerId ProviderIdPath - - err = runtime.BindStyledParameterWithOptions("simple", "providerId", chi.URLParam(r, "providerId"), &providerId, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true, Type: "string", Format: ""}) - if err != nil { - siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "providerId", Err: err}) - return - } - - ctx := r.Context() - - ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) - - r = r.WithContext(ctx) - - handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - siw.Handler.DeleteProvider(w, r, providerId) - })) - - for _, middleware := range siw.HandlerMiddlewares { - handler = middleware(handler) - } - - handler.ServeHTTP(w, r) -} - -// GetProvider operation middleware -func (siw *ServerInterfaceWrapper) GetProvider(w http.ResponseWriter, r *http.Request) { - - var err error - _ = err - - // ------------- Path parameter "providerId" ------------- - var providerId ProviderIdPath - - err = runtime.BindStyledParameterWithOptions("simple", "providerId", chi.URLParam(r, "providerId"), &providerId, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true, Type: "string", Format: ""}) - if err != nil { - siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "providerId", Err: err}) - return - } - - ctx := r.Context() - - ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) - - r = r.WithContext(ctx) - - handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - siw.Handler.GetProvider(w, r, providerId) - })) - - for _, middleware := range siw.HandlerMiddlewares { - handler = middleware(handler) - } - - handler.ServeHTTP(w, r) -} - -// ApplyProvider operation middleware -func (siw *ServerInterfaceWrapper) ApplyProvider(w http.ResponseWriter, r *http.Request) { - - var err error - _ = err - - // ------------- Path parameter "providerId" ------------- - var providerId ProviderIdPath - - err = runtime.BindStyledParameterWithOptions("simple", "providerId", chi.URLParam(r, "providerId"), &providerId, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true, Type: "string", Format: ""}) - if err != nil { - siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "providerId", Err: err}) - return - } - - ctx := r.Context() - - ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) - - r = r.WithContext(ctx) - - handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - siw.Handler.ApplyProvider(w, r, providerId) - })) - - for _, middleware := range siw.HandlerMiddlewares { - handler = middleware(handler) - } - - handler.ServeHTTP(w, r) -} - -type UnescapedCookieParamError struct { - ParamName string - Err error -} - -func (e *UnescapedCookieParamError) Error() string { - return fmt.Sprintf("error unescaping cookie parameter '%s'", e.ParamName) -} - -func (e *UnescapedCookieParamError) Unwrap() error { - return e.Err -} - -type UnmarshalingParamError struct { - ParamName string - Err error -} - -func (e *UnmarshalingParamError) Error() string { - return fmt.Sprintf("Error unmarshaling parameter %s as JSON: %s", e.ParamName, e.Err.Error()) -} - -func (e *UnmarshalingParamError) Unwrap() error { - return e.Err -} - -type RequiredParamError struct { - ParamName string -} - -func (e *RequiredParamError) Error() string { - return fmt.Sprintf("Query argument %s is required, but not found", e.ParamName) -} - -type RequiredHeaderError struct { - ParamName string - Err error -} - -func (e *RequiredHeaderError) Error() string { - return fmt.Sprintf("Header parameter %s is required, but not found", e.ParamName) -} - -func (e *RequiredHeaderError) Unwrap() error { - return e.Err -} - -type InvalidParamFormatError struct { - ParamName string - Err error -} - -func (e *InvalidParamFormatError) Error() string { - return fmt.Sprintf("Invalid format for parameter %s: %s", e.ParamName, e.Err.Error()) -} - -func (e *InvalidParamFormatError) Unwrap() error { - return e.Err -} - -type TooManyValuesForParamError struct { - ParamName string - Count int -} - -func (e *TooManyValuesForParamError) Error() string { - return fmt.Sprintf("Expected one value for %s, got %d", e.ParamName, e.Count) -} - -// Handler creates http.Handler with routing matching OpenAPI spec. -func Handler(si ServerInterface) http.Handler { - return HandlerWithOptions(si, ChiServerOptions{}) -} - -type ChiServerOptions struct { - BaseURL string - BaseRouter chi.Router - Middlewares []MiddlewareFunc - ErrorHandlerFunc func(w http.ResponseWriter, r *http.Request, err error) -} - -// HandlerFromMux creates http.Handler with routing matching OpenAPI spec based on the provided mux. -func HandlerFromMux(si ServerInterface, r chi.Router) http.Handler { - return HandlerWithOptions(si, ChiServerOptions{ - BaseRouter: r, - }) -} - -func HandlerFromMuxWithBaseURL(si ServerInterface, r chi.Router, baseURL string) http.Handler { - return HandlerWithOptions(si, ChiServerOptions{ - BaseURL: baseURL, - BaseRouter: r, - }) -} - -// HandlerWithOptions creates http.Handler with additional options -func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handler { - r := options.BaseRouter - - if r == nil { - r = chi.NewRouter() - } - if options.ErrorHandlerFunc == nil { - options.ErrorHandlerFunc = func(w http.ResponseWriter, r *http.Request, err error) { - http.Error(w, err.Error(), http.StatusBadRequest) - } - } - wrapper := ServerInterfaceWrapper{ - Handler: si, - HandlerMiddlewares: options.Middlewares, - ErrorHandlerFunc: options.ErrorHandlerFunc, - } - - r.Group(func(r chi.Router) { - r.Get(options.BaseURL+"/providers", wrapper.ListProviders) - }) - r.Group(func(r chi.Router) { - r.Post(options.BaseURL+"/providers", wrapper.CreateProvider) - }) - r.Group(func(r chi.Router) { - r.Delete(options.BaseURL+"/providers/{providerId}", wrapper.DeleteProvider) - }) - r.Group(func(r chi.Router) { - r.Get(options.BaseURL+"/providers/{providerId}", wrapper.GetProvider) - }) - r.Group(func(r chi.Router) { - r.Put(options.BaseURL+"/providers/{providerId}", wrapper.ApplyProvider) - }) - - return r -} - -type ForbiddenJSONResponse Error - -type UnauthorizedJSONResponse Error - -type ListProvidersRequestObject struct { - Params ListProvidersParams -} - -type ListProvidersResponseObject interface { - VisitListProvidersResponse(w http.ResponseWriter) error -} - -type ListProviders200JSONResponse ProviderList - -func (response ListProviders200JSONResponse) VisitListProvidersResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(200) - _, err := buf.WriteTo(w) - return err -} - -type ListProviders400ApplicationProblemPlusJSONResponse Error - -func (response ListProviders400ApplicationProblemPlusJSONResponse) VisitListProvidersResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(400) - _, err := buf.WriteTo(w) - return err -} - -type ListProviders401JSONResponse struct{ UnauthorizedJSONResponse } - -func (response ListProviders401JSONResponse) VisitListProvidersResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(401) - _, err := buf.WriteTo(w) - return err -} - -type ListProviders403JSONResponse struct{ ForbiddenJSONResponse } - -func (response ListProviders403JSONResponse) VisitListProvidersResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(403) - _, err := buf.WriteTo(w) - return err -} - -type ListProvidersdefaultApplicationProblemPlusJSONResponse struct { - Body Error - StatusCode int -} - -func (response ListProvidersdefaultApplicationProblemPlusJSONResponse) VisitListProvidersResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response.Body); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(response.StatusCode) - _, err := buf.WriteTo(w) - return err -} - -type CreateProviderRequestObject struct { - Params CreateProviderParams - Body *CreateProviderJSONRequestBody -} - -type CreateProviderResponseObject interface { - VisitCreateProviderResponse(w http.ResponseWriter) error -} - -type CreateProvider200JSONResponse Provider - -func (response CreateProvider200JSONResponse) VisitCreateProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(200) - _, err := buf.WriteTo(w) - return err -} - -type CreateProvider201JSONResponse Provider - -func (response CreateProvider201JSONResponse) VisitCreateProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(201) - _, err := buf.WriteTo(w) - return err -} - -type CreateProvider400ApplicationProblemPlusJSONResponse Error - -func (response CreateProvider400ApplicationProblemPlusJSONResponse) VisitCreateProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(400) - _, err := buf.WriteTo(w) - return err -} - -type CreateProvider401JSONResponse struct{ UnauthorizedJSONResponse } - -func (response CreateProvider401JSONResponse) VisitCreateProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(401) - _, err := buf.WriteTo(w) - return err -} - -type CreateProvider403JSONResponse struct{ ForbiddenJSONResponse } - -func (response CreateProvider403JSONResponse) VisitCreateProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(403) - _, err := buf.WriteTo(w) - return err -} - -type CreateProvider409ApplicationProblemPlusJSONResponse Error - -func (response CreateProvider409ApplicationProblemPlusJSONResponse) VisitCreateProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(409) - _, err := buf.WriteTo(w) - return err -} - -type CreateProvider422ApplicationProblemPlusJSONResponse Error - -func (response CreateProvider422ApplicationProblemPlusJSONResponse) VisitCreateProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(422) - _, err := buf.WriteTo(w) - return err -} - -type CreateProviderdefaultApplicationProblemPlusJSONResponse struct { - Body Error - StatusCode int -} - -func (response CreateProviderdefaultApplicationProblemPlusJSONResponse) VisitCreateProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response.Body); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(response.StatusCode) - _, err := buf.WriteTo(w) - return err -} - -type DeleteProviderRequestObject struct { - ProviderId ProviderIdPath `json:"providerId"` -} - -type DeleteProviderResponseObject interface { - VisitDeleteProviderResponse(w http.ResponseWriter) error -} - -type DeleteProvider204Response struct { -} - -func (response DeleteProvider204Response) VisitDeleteProviderResponse(w http.ResponseWriter) error { - w.WriteHeader(204) - return nil -} - -type DeleteProvider400ApplicationProblemPlusJSONResponse Error - -func (response DeleteProvider400ApplicationProblemPlusJSONResponse) VisitDeleteProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(400) - _, err := buf.WriteTo(w) - return err -} - -type DeleteProvider401JSONResponse struct{ UnauthorizedJSONResponse } - -func (response DeleteProvider401JSONResponse) VisitDeleteProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(401) - _, err := buf.WriteTo(w) - return err -} - -type DeleteProvider403JSONResponse struct{ ForbiddenJSONResponse } - -func (response DeleteProvider403JSONResponse) VisitDeleteProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(403) - _, err := buf.WriteTo(w) - return err -} - -type DeleteProvider404ApplicationProblemPlusJSONResponse Error - -func (response DeleteProvider404ApplicationProblemPlusJSONResponse) VisitDeleteProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(404) - _, err := buf.WriteTo(w) - return err -} - -type DeleteProviderdefaultApplicationProblemPlusJSONResponse struct { - Body Error - StatusCode int -} - -func (response DeleteProviderdefaultApplicationProblemPlusJSONResponse) VisitDeleteProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response.Body); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(response.StatusCode) - _, err := buf.WriteTo(w) - return err -} - -type GetProviderRequestObject struct { - ProviderId ProviderIdPath `json:"providerId"` -} - -type GetProviderResponseObject interface { - VisitGetProviderResponse(w http.ResponseWriter) error -} - -type GetProvider200JSONResponse Provider - -func (response GetProvider200JSONResponse) VisitGetProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(200) - _, err := buf.WriteTo(w) - return err -} - -type GetProvider400ApplicationProblemPlusJSONResponse Error - -func (response GetProvider400ApplicationProblemPlusJSONResponse) VisitGetProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(400) - _, err := buf.WriteTo(w) - return err -} - -type GetProvider401JSONResponse struct{ UnauthorizedJSONResponse } - -func (response GetProvider401JSONResponse) VisitGetProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(401) - _, err := buf.WriteTo(w) - return err -} - -type GetProvider403JSONResponse struct{ ForbiddenJSONResponse } - -func (response GetProvider403JSONResponse) VisitGetProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(403) - _, err := buf.WriteTo(w) - return err -} - -type GetProvider404ApplicationProblemPlusJSONResponse Error - -func (response GetProvider404ApplicationProblemPlusJSONResponse) VisitGetProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(404) - _, err := buf.WriteTo(w) - return err -} - -type GetProviderdefaultApplicationProblemPlusJSONResponse struct { - Body Error - StatusCode int -} - -func (response GetProviderdefaultApplicationProblemPlusJSONResponse) VisitGetProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response.Body); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(response.StatusCode) - _, err := buf.WriteTo(w) - return err -} - -type ApplyProviderRequestObject struct { - ProviderId ProviderIdPath `json:"providerId"` - Body *ApplyProviderJSONRequestBody -} - -type ApplyProviderResponseObject interface { - VisitApplyProviderResponse(w http.ResponseWriter) error -} - -type ApplyProvider200JSONResponse Provider - -func (response ApplyProvider200JSONResponse) VisitApplyProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(200) - _, err := buf.WriteTo(w) - return err -} - -type ApplyProvider400ApplicationProblemPlusJSONResponse Error - -func (response ApplyProvider400ApplicationProblemPlusJSONResponse) VisitApplyProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(400) - _, err := buf.WriteTo(w) - return err -} - -type ApplyProvider401JSONResponse struct{ UnauthorizedJSONResponse } - -func (response ApplyProvider401JSONResponse) VisitApplyProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(401) - _, err := buf.WriteTo(w) - return err -} - -type ApplyProvider403JSONResponse struct{ ForbiddenJSONResponse } - -func (response ApplyProvider403JSONResponse) VisitApplyProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(403) - _, err := buf.WriteTo(w) - return err -} - -type ApplyProvider404ApplicationProblemPlusJSONResponse Error - -func (response ApplyProvider404ApplicationProblemPlusJSONResponse) VisitApplyProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(404) - _, err := buf.WriteTo(w) - return err -} - -type ApplyProvider409ApplicationProblemPlusJSONResponse Error - -func (response ApplyProvider409ApplicationProblemPlusJSONResponse) VisitApplyProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(409) - _, err := buf.WriteTo(w) - return err -} - -type ApplyProviderdefaultApplicationProblemPlusJSONResponse struct { - Body Error - StatusCode int -} - -func (response ApplyProviderdefaultApplicationProblemPlusJSONResponse) VisitApplyProviderResponse(w http.ResponseWriter) error { - - var buf bytes.Buffer - if err := json.NewEncoder(&buf).Encode(response.Body); err != nil { - return err - } - w.Header().Set("Content-Type", "application/problem+json") - w.WriteHeader(response.StatusCode) - _, err := buf.WriteTo(w) - return err -} - -// StrictServerInterface represents all server handlers. -type StrictServerInterface interface { - // List all providers - // (GET /providers) - ListProviders(ctx context.Context, request ListProvidersRequestObject) (ListProvidersResponseObject, error) - // Register a Service Provider - // (POST /providers) - CreateProvider(ctx context.Context, request CreateProviderRequestObject) (CreateProviderResponseObject, error) - // Delete a service Provider - // (DELETE /providers/{providerId}) - DeleteProvider(ctx context.Context, request DeleteProviderRequestObject) (DeleteProviderResponseObject, error) - // Get a provider - // (GET /providers/{providerId}) - GetProvider(ctx context.Context, request GetProviderRequestObject) (GetProviderResponseObject, error) - // Update a Service Provider - // (PUT /providers/{providerId}) - ApplyProvider(ctx context.Context, request ApplyProviderRequestObject) (ApplyProviderResponseObject, error) -} - -type StrictHandlerFunc func(ctx context.Context, w http.ResponseWriter, r *http.Request, request any) (any, error) -type StrictMiddlewareFunc func(f StrictHandlerFunc, operationID string) StrictHandlerFunc - -type StrictHTTPServerOptions struct { - RequestErrorHandlerFunc func(w http.ResponseWriter, r *http.Request, err error) - ResponseErrorHandlerFunc func(w http.ResponseWriter, r *http.Request, err error) -} - -func NewStrictHandler(ssi StrictServerInterface, middlewares []StrictMiddlewareFunc) ServerInterface { - return &strictHandler{ssi: ssi, middlewares: middlewares, options: StrictHTTPServerOptions{ - RequestErrorHandlerFunc: func(w http.ResponseWriter, r *http.Request, err error) { - http.Error(w, err.Error(), http.StatusBadRequest) - }, - ResponseErrorHandlerFunc: func(w http.ResponseWriter, r *http.Request, err error) { - http.Error(w, err.Error(), http.StatusInternalServerError) - }, - }} -} - -func NewStrictHandlerWithOptions(ssi StrictServerInterface, middlewares []StrictMiddlewareFunc, options StrictHTTPServerOptions) ServerInterface { - return &strictHandler{ssi: ssi, middlewares: middlewares, options: options} -} - -type strictHandler struct { - ssi StrictServerInterface - middlewares []StrictMiddlewareFunc - options StrictHTTPServerOptions -} - -// ListProviders operation middleware -func (sh *strictHandler) ListProviders(w http.ResponseWriter, r *http.Request, params ListProvidersParams) { - var request ListProvidersRequestObject - - request.Params = params - - handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { - return sh.ssi.ListProviders(ctx, request.(ListProvidersRequestObject)) - } - for _, middleware := range sh.middlewares { - handler = middleware(handler, "ListProviders") - } - - response, err := handler(r.Context(), w, r, request) - - if err != nil { - sh.options.ResponseErrorHandlerFunc(w, r, err) - } else if validResponse, ok := response.(ListProvidersResponseObject); ok { - if err := validResponse.VisitListProvidersResponse(w); err != nil { - sh.options.ResponseErrorHandlerFunc(w, r, err) - } - } else if response != nil { - sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) - } -} - -// CreateProvider operation middleware -func (sh *strictHandler) CreateProvider(w http.ResponseWriter, r *http.Request, params CreateProviderParams) { - var request CreateProviderRequestObject - - request.Params = params - - var body CreateProviderJSONRequestBody - if err := json.NewDecoder(r.Body).Decode(&body); err != nil { - sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) - return - } - request.Body = &body - - handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { - return sh.ssi.CreateProvider(ctx, request.(CreateProviderRequestObject)) - } - for _, middleware := range sh.middlewares { - handler = middleware(handler, "CreateProvider") - } - - response, err := handler(r.Context(), w, r, request) - - if err != nil { - sh.options.ResponseErrorHandlerFunc(w, r, err) - } else if validResponse, ok := response.(CreateProviderResponseObject); ok { - if err := validResponse.VisitCreateProviderResponse(w); err != nil { - sh.options.ResponseErrorHandlerFunc(w, r, err) - } - } else if response != nil { - sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) - } -} - -// DeleteProvider operation middleware -func (sh *strictHandler) DeleteProvider(w http.ResponseWriter, r *http.Request, providerId ProviderIdPath) { - var request DeleteProviderRequestObject - - request.ProviderId = providerId - - handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { - return sh.ssi.DeleteProvider(ctx, request.(DeleteProviderRequestObject)) - } - for _, middleware := range sh.middlewares { - handler = middleware(handler, "DeleteProvider") - } - - response, err := handler(r.Context(), w, r, request) - - if err != nil { - sh.options.ResponseErrorHandlerFunc(w, r, err) - } else if validResponse, ok := response.(DeleteProviderResponseObject); ok { - if err := validResponse.VisitDeleteProviderResponse(w); err != nil { - sh.options.ResponseErrorHandlerFunc(w, r, err) - } - } else if response != nil { - sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) - } -} - -// GetProvider operation middleware -func (sh *strictHandler) GetProvider(w http.ResponseWriter, r *http.Request, providerId ProviderIdPath) { - var request GetProviderRequestObject - - request.ProviderId = providerId - - handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { - return sh.ssi.GetProvider(ctx, request.(GetProviderRequestObject)) - } - for _, middleware := range sh.middlewares { - handler = middleware(handler, "GetProvider") - } - - response, err := handler(r.Context(), w, r, request) - - if err != nil { - sh.options.ResponseErrorHandlerFunc(w, r, err) - } else if validResponse, ok := response.(GetProviderResponseObject); ok { - if err := validResponse.VisitGetProviderResponse(w); err != nil { - sh.options.ResponseErrorHandlerFunc(w, r, err) - } - } else if response != nil { - sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) - } -} - -// ApplyProvider operation middleware -func (sh *strictHandler) ApplyProvider(w http.ResponseWriter, r *http.Request, providerId ProviderIdPath) { - var request ApplyProviderRequestObject - - request.ProviderId = providerId - - var body ApplyProviderJSONRequestBody - if err := json.NewDecoder(r.Body).Decode(&body); err != nil { - sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) - return - } - request.Body = &body - - handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { - return sh.ssi.ApplyProvider(ctx, request.(ApplyProviderRequestObject)) - } - for _, middleware := range sh.middlewares { - handler = middleware(handler, "ApplyProvider") - } - - response, err := handler(r.Context(), w, r, request) - - if err != nil { - sh.options.ResponseErrorHandlerFunc(w, r, err) - } else if validResponse, ok := response.(ApplyProviderResponseObject); ok { - if err := validResponse.VisitApplyProviderResponse(w); err != nil { - sh.options.ResponseErrorHandlerFunc(w, r, err) - } - } else if response != nil { - sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) - } -} diff --git a/internal/sp/api/resource_manager/server.gen.go b/internal/sp/api/resource_manager/server.gen.go index 72bde3f..b635b64 100644 --- a/internal/sp/api/resource_manager/server.gen.go +++ b/internal/sp/api/resource_manager/server.gen.go @@ -22,9 +22,8 @@ const ( // Defines values for ServiceTypeInstanceDeletionStatus. const ( - FAILED ServiceTypeInstanceDeletionStatus = "FAILED" - PENDINGPROVIDER ServiceTypeInstanceDeletionStatus = "PENDING_PROVIDER" - SCHEDULED ServiceTypeInstanceDeletionStatus = "SCHEDULED" + FAILED ServiceTypeInstanceDeletionStatus = "FAILED" + SCHEDULED ServiceTypeInstanceDeletionStatus = "SCHEDULED" ) // Valid indicates whether the value is a known member of the ServiceTypeInstanceDeletionStatus enum. @@ -32,8 +31,6 @@ func (e ServiceTypeInstanceDeletionStatus) Valid() bool { switch e { case FAILED: return true - case PENDINGPROVIDER: - return true case SCHEDULED: return true default: @@ -61,14 +58,16 @@ type Error struct { // ServiceTypeInstance Full service type instance resource representation type ServiceTypeInstance struct { + // AgentName Name of the agent managing this instance. Absent or null when the + // instance was created without agent routing. + AgentName *string `json:"agent_name,omitempty"` + // CreateTime Timestamp when the instance was first created CreateTime *time.Time `json:"create_time,omitempty"` // DeletionStatus Deletion status for deferred deletions. Absent for active // instances. SCHEDULED indicates the instance is queued for cleanup. // FAILED indicates the cleanup has exceeded maximum retries. - // PENDING_PROVIDER indicates the instance is waiting for its - // provider to become healthy before cleanup is retried. DeletionStatus *ServiceTypeInstanceDeletionStatus `json:"deletion_status,omitempty"` // Id Unique identifier for the Service Type Instance @@ -77,9 +76,6 @@ type ServiceTypeInstance struct { // Path Resource path identifier Path *string `json:"path,omitempty"` - // ProviderName Name of the provider - ProviderName string `json:"provider_name"` - // Spec Service specification following one of the supported service type // schemas (VMSpec, ContainerSpec, DatabaseSpec, or ClusterSpec). Spec map[string]interface{} `json:"spec"` @@ -94,8 +90,6 @@ type ServiceTypeInstance struct { // ServiceTypeInstanceDeletionStatus Deletion status for deferred deletions. Absent for active // instances. SCHEDULED indicates the instance is queued for cleanup. // FAILED indicates the cleanup has exceeded maximum retries. -// PENDING_PROVIDER indicates the instance is waiting for its -// provider to become healthy before cleanup is retried. type ServiceTypeInstanceDeletionStatus string // ServiceTypeInstanceList Paginated list of instances @@ -120,12 +114,12 @@ type bearerAuthContextKey string // ListInstancesParams defines parameters for ListInstances. type ListInstancesParams struct { - // Provider Filter service provider - Provider *string `form:"provider,omitempty" json:"provider,omitempty"` - // ServiceType Filter instances by service type ServiceType *string `form:"service_type,omitempty" json:"service_type,omitempty"` + // AgentName Filter instances by the agent managing them + AgentName *string `form:"agent_name,omitempty" json:"agent_name,omitempty"` + // ShowDeleted If true, soft-deleted instances are included in the results // alongside active instances. Defaults to false. ShowDeleted *bool `form:"show_deleted,omitempty" json:"show_deleted,omitempty"` @@ -228,28 +222,28 @@ func (siw *ServerInterfaceWrapper) ListInstances(w http.ResponseWriter, r *http. // Parameter object where we will unmarshal all parameters from the context var params ListInstancesParams - // ------------- Optional query parameter "provider" ------------- + // ------------- Optional query parameter "service_type" ------------- - err = runtime.BindQueryParameterWithOptions("form", true, false, "provider", r.URL.Query(), ¶ms.Provider, runtime.BindQueryParameterOptions{Type: "string", Format: ""}) + err = runtime.BindQueryParameterWithOptions("form", true, false, "service_type", r.URL.Query(), ¶ms.ServiceType, runtime.BindQueryParameterOptions{Type: "string", Format: ""}) if err != nil { var requiredError *runtime.RequiredParameterError if errors.As(err, &requiredError) { - siw.ErrorHandlerFunc(w, r, &RequiredParamError{ParamName: "provider"}) + siw.ErrorHandlerFunc(w, r, &RequiredParamError{ParamName: "service_type"}) } else { - siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "provider", Err: err}) + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "service_type", Err: err}) } return } - // ------------- Optional query parameter "service_type" ------------- + // ------------- Optional query parameter "agent_name" ------------- - err = runtime.BindQueryParameterWithOptions("form", true, false, "service_type", r.URL.Query(), ¶ms.ServiceType, runtime.BindQueryParameterOptions{Type: "string", Format: ""}) + err = runtime.BindQueryParameterWithOptions("form", true, false, "agent_name", r.URL.Query(), ¶ms.AgentName, runtime.BindQueryParameterOptions{Type: "string", Format: ""}) if err != nil { var requiredError *runtime.RequiredParameterError if errors.As(err, &requiredError) { - siw.ErrorHandlerFunc(w, r, &RequiredParamError{ParamName: "service_type"}) + siw.ErrorHandlerFunc(w, r, &RequiredParamError{ParamName: "agent_name"}) } else { - siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "service_type", Err: err}) + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "agent_name", Err: err}) } return } @@ -760,6 +754,20 @@ func (response CreateInstance422ApplicationProblemPlusJSONResponse) VisitCreateI return err } +type CreateInstance503ApplicationProblemPlusJSONResponse Error + +func (response CreateInstance503ApplicationProblemPlusJSONResponse) VisitCreateInstanceResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(503) + _, err := buf.WriteTo(w) + return err +} + type CreateInstancedefaultApplicationProblemPlusJSONResponse struct { Body Error StatusCode int @@ -850,6 +858,20 @@ func (response DeleteInstance404ApplicationProblemPlusJSONResponse) VisitDeleteI return err } +type DeleteInstance422ApplicationProblemPlusJSONResponse Error + +func (response DeleteInstance422ApplicationProblemPlusJSONResponse) VisitDeleteInstanceResponse(w http.ResponseWriter) error { + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(response); err != nil { + return err + } + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(422) + _, err := buf.WriteTo(w) + return err +} + type DeleteInstancedefaultApplicationProblemPlusJSONResponse struct { Body Error StatusCode int diff --git a/internal/sp/cleanup/scheduler.go b/internal/sp/cleanup/scheduler.go index d9bbca7..af9b4fd 100644 --- a/internal/sp/cleanup/scheduler.go +++ b/internal/sp/cleanup/scheduler.go @@ -1,48 +1,51 @@ -// Package cleanup implements a scheduler for deferred deletions of service type instances. +// Package cleanup implements stale instance cleanup scheduling. package cleanup import ( "context" + "errors" "sync" "time" + agentstore "github.com/dcm-project/control-plane/internal/agent/store/agent" "github.com/dcm-project/control-plane/internal/sp/config" "github.com/dcm-project/control-plane/internal/sp/logging" - rmsvc "github.com/dcm-project/control-plane/internal/sp/service/resource_manager" + "github.com/dcm-project/control-plane/internal/sp/messaging" "github.com/dcm-project/control-plane/internal/sp/store" "github.com/dcm-project/control-plane/internal/sp/store/model" ) -// Scheduler periodically retries deferred deletions of service type instances. type Scheduler struct { - store store.Store - instanceService *rmsvc.InstanceService - interval time.Duration - maxRetries int - stopCh chan struct{} - wg sync.WaitGroup + store store.Store + publisher *messaging.Publisher + agentStore agentstore.Agent + interval time.Duration + timeout time.Duration + maxRetries int + stopCh chan struct{} + stopOnce sync.Once + wg sync.WaitGroup } -// NewScheduler creates a new cleanup scheduler. -func NewScheduler(store store.Store, instanceService *rmsvc.InstanceService, cfg *config.CleanupConfig) *Scheduler { +func NewScheduler(store store.Store, publisher *messaging.Publisher, agentSt agentstore.Agent, cfg *config.CleanupConfig) *Scheduler { return &Scheduler{ - store: store, - instanceService: instanceService, - interval: cfg.Interval, - maxRetries: cfg.MaxRetries, - stopCh: make(chan struct{}), + store: store, + publisher: publisher, + agentStore: agentSt, + interval: cfg.Interval, + timeout: cfg.Timeout, + maxRetries: cfg.MaxRetries, + stopCh: make(chan struct{}), } } -// Start begins the cleanup scheduling loop. func (s *Scheduler) Start(ctx context.Context) { s.wg.Add(1) go s.run(ctx) } -// Stop gracefully stops the cleanup scheduler. func (s *Scheduler) Stop() { - close(s.stopCh) + s.stopOnce.Do(func() { close(s.stopCh) }) s.wg.Wait() } @@ -59,12 +62,23 @@ func (s *Scheduler) run(ctx context.Context) { case <-s.stopCh: return case <-ticker.C: - s.ProcessPendingDeletions(ctx) + s.runCycle(ctx) } } } -// ProcessPendingDeletions attempts to complete all deferred deletions. +// runCycle bounds a single sweep of pending deletions to the configured +// timeout so a slow DB or agent lookup can't stall the next tick indefinitely. +func (s *Scheduler) runCycle(ctx context.Context) { + if s.timeout <= 0 { + s.ProcessPendingDeletions(ctx) + return + } + cycleCtx, cancel := context.WithTimeout(ctx, s.timeout) + defer cancel() + s.ProcessPendingDeletions(cycleCtx) +} + func (s *Scheduler) ProcessPendingDeletions(ctx context.Context) { log := logging.FromContext(ctx) pending, err := s.store.ServiceTypeInstance().ListPendingDeletions(ctx) @@ -83,39 +97,75 @@ func (s *Scheduler) ProcessPendingDeletions(ctx context.Context) { } } +// processOne advances one deferred deletion by one step. An agent-routed +// instance is only marked DELETED once its "deletion-acknowledged" event +// arrives (see consumer.ResponseConsumer), except for the audited give-up +// cases below. func (s *Scheduler) processOne(ctx context.Context, instance model.ServiceTypeInstance) { log := logging.FromContext(ctx) - marked, err := s.store.ServiceTypeInstance().MarkPendingProviderIfNotReady(ctx, instance.ID) - if err != nil { - log.Error("Failed to check provider health for instance", "instance_id", instance.ID, "error", err) + if instance.AgentName == nil { + // Never agent-routed: there is no physical resource on an agent to + // wait for, so this is a normal (non-audited) completion. + log.Info("cleanup: no agent, marking DELETED", "instance_id", instance.ID) + if err := s.store.ServiceTypeInstance().MarkDeletionComplete(ctx, instance.ID); err != nil { + log.Error("Failed to mark instance as DELETED", "instance_id", instance.ID, "error", err) + } return } - if marked { - log.Info("Instance marked as PENDING_PROVIDER, provider is not ready", "instance_id", instance.ID, "provider_name", instance.ProviderName) + + if s.publisher == nil || s.agentStore == nil { + s.auditGiveUp(ctx, instance, "publisher_or_agent_store_unavailable") return } - if err := s.instanceService.DeleteFromProvider(ctx, &instance); err != nil { - log.Error("Failed to delete instance from provider", "instance_id", instance.ID, "provider_name", instance.ProviderName, "error", err) - s.handleRetryOrFail(ctx, instance) + agent, err := s.agentStore.GetByName(ctx, *instance.AgentName) + if err != nil { + if errors.Is(err, agentstore.ErrAgentNotFound) { + s.auditGiveUp(ctx, instance, "agent_not_found") + return + } + log.Error("cleanup: agent lookup failed, will retry next cycle", "instance_id", instance.ID, "error", err) return } - log.Info("Successfully deleted instance from provider", "instance_id", instance.ID, "provider_name", instance.ProviderName) -} -func (s *Scheduler) handleRetryOrFail(ctx context.Context, instance model.ServiceTypeInstance) { - log := logging.FromContext(ctx) - if instance.RetryCount+1 >= s.maxRetries { + if s.maxRetries > 0 && instance.RetryCount >= s.maxRetries { + log.Warn("cleanup audit: deletion retries exhausted, marking FAILED for manual intervention", + "instance_id", instance.ID, "agent_name", *instance.AgentName, "retry_count", instance.RetryCount, "reason", "retries_exhausted") if err := s.store.ServiceTypeInstance().MarkDeletionFailed(ctx, instance.ID); err != nil { - log.Error("Failed to mark instance as FAILED", "instance_id", instance.ID, "error", err) - } else { - log.Warn("Instance exceeded max retries, marked as FAILED", "instance_id", instance.ID, "max_retries", s.maxRetries) + log.Error("Failed to mark instance deletion as FAILED", "instance_id", instance.ID, "error", err) } return } + pubErr := s.publisher.PublishDelete(ctx, agent.TopicName, messaging.DeletePayload{ + ResourceID: instance.ID, + ServiceType: instance.ServiceType, + }) + if pubErr != nil { + log.Warn("cleanup: delete publish failed, will retry next cycle", "instance_id", instance.ID, "error", pubErr) + } else { + log.Info("cleanup: delete published, awaiting agent acknowledgement", "instance_id", instance.ID) + } + + // Every attempt counts toward maxRetries whether or not the publish + // itself succeeded, so a permanently unreachable NATS/agent eventually + // trips the retries-exhausted branch above instead of retrying forever. if err := s.store.ServiceTypeInstance().IncrementDeletionRetry(ctx, instance.ID); err != nil { - log.Error("Failed to increment retry count for instance", "instance_id", instance.ID, "error", err) + log.Error("Failed to record deletion retry attempt", "instance_id", instance.ID, "error", err) + } +} + +// auditGiveUp marks an instance DELETED without ever confirming the physical +// resource was removed, because the CP has lost its only path to ask the +// agent (agent deregistered, or no NATS/agent store wired up). This is +// intentionally logged at Warn with a structured reason so operators can +// find instances whose backing resource may be orphaned (REQ-CLEANUP-AUDIT). +func (s *Scheduler) auditGiveUp(ctx context.Context, instance model.ServiceTypeInstance, reason string) { + log := logging.FromContext(ctx) + log.Warn("cleanup audit: marking DELETED without confirmed physical deletion", + "instance_id", instance.ID, "agent_name", *instance.AgentName, "reason", reason) + if err := s.store.ServiceTypeInstance().MarkDeletionComplete(ctx, instance.ID); err != nil { + log.Error("Failed to mark instance as DELETED", "instance_id", instance.ID, "error", err) } } diff --git a/internal/sp/cleanup/scheduler_test.go b/internal/sp/cleanup/scheduler_test.go index d2cf2e6..eabfa4e 100644 --- a/internal/sp/cleanup/scheduler_test.go +++ b/internal/sp/cleanup/scheduler_test.go @@ -2,18 +2,18 @@ package cleanup_test import ( "context" - "net/http" - "net/http/httptest" "time" + agentstore "github.com/dcm-project/control-plane/internal/agent/store/agent" + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" "github.com/dcm-project/control-plane/internal/sp/cleanup" "github.com/dcm-project/control-plane/internal/sp/config" - rmsvc "github.com/dcm-project/control-plane/internal/sp/service/resource_manager" + "github.com/dcm-project/control-plane/internal/sp/messaging" "github.com/dcm-project/control-plane/internal/sp/store" "github.com/dcm-project/control-plane/internal/sp/store/model" "github.com/dcm-project/control-plane/internal/sp/testutil" - "github.com/go-resty/resty/v2" "github.com/google/uuid" + "github.com/nats-io/nats.go/jetstream" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "gorm.io/driver/sqlite" @@ -21,15 +21,26 @@ import ( "gorm.io/gorm/logger" ) +// stubJetStream acknowledges every publish so tests can exercise the +// publish-then-wait-for-ack path without a real NATS server. +type stubJetStream struct { + jetstream.JetStream + publishErr error +} + +func (s *stubJetStream) Publish(_ context.Context, _ string, _ []byte, _ ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + if s.publishErr != nil { + return nil, s.publishErr + } + return &jetstream.PubAck{}, nil +} + var _ = Describe("Scheduler", func() { var ( - db *gorm.DB - dataStore store.Store - instanceService *rmsvc.InstanceService - scheduler *cleanup.Scheduler - ctx context.Context - mockProvider *httptest.Server - providerName string + db *gorm.DB + dataStore store.Store + scheduler *cleanup.Scheduler + ctx context.Context ) BeforeEach(func() { @@ -38,85 +49,58 @@ var _ = Describe("Scheduler", func() { Logger: logger.Default.LogMode(logger.Silent), }) Expect(err).NotTo(HaveOccurred()) - Expect(db.AutoMigrate(&model.Provider{}, &model.ServiceTypeInstance{})).To(Succeed()) - - providerName = "test-provider" + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) - // Mock provider server that returns 204 on DELETE - mockProvider = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodDelete { - w.WriteHeader(http.StatusNoContent) - return - } - w.WriteHeader(http.StatusOK) - })) - - // Create a provider in the database - provider := model.Provider{ - ID: uuid.New().String(), - Name: providerName, - ServiceType: "vm", - SchemaVersion: "v1", - Endpoint: mockProvider.URL, - HealthStatus: model.HealthStatusReady, + for _, name := range []string{"audit-agent", "unregistered-agent", "mismatch-agent"} { + Expect(db.Create(&agentmodel.Agent{ID: uuid.New().String(), Name: name, TopicName: "dcm.agent." + name}).Error).NotTo(HaveOccurred()) } - Expect(db.Create(&provider).Error).NotTo(HaveOccurred()) dataStore = store.NewStore(db, store.WithServiceTypeInstanceRetry(testutil.FastServiceTypeInstanceRetry()...)) - instanceService = rmsvc.NewInstanceService(dataStore, resty.New(). - SetTimeout(5*time.Second). - SetRetryCount(0)) cfg := &config.CleanupConfig{ Interval: 1 * time.Minute, MaxRetries: 3, } - scheduler = cleanup.NewScheduler(dataStore, instanceService, cfg) + scheduler = cleanup.NewScheduler(dataStore, nil, nil, cfg) ctx = context.Background() }) AfterEach(func() { - mockProvider.Close() sqlDB, err := db.DB() Expect(err).NotTo(HaveOccurred()) Expect(sqlDB.Close()).To(Succeed()) }) - Describe("ProcessPendingDeletions with provider health check", func() { - It("skips instance when provider is not ready", func() { - // Mark provider as NotReady - Expect(db.Model(&model.Provider{}).Where("name = ?", providerName). - Update("health_status", model.HealthStatusUnavailable).Error).NotTo(HaveOccurred()) - - // Create an instance marked for deletion + Describe("Agent-only cleanup (all instances are agent-routed)", func() { + It("marks DELETED for agent-routed instance", func() { + agentName := "audit-agent" inst := model.ServiceTypeInstance{ ID: uuid.New().String(), - ProviderName: providerName, + ServiceType: "vm", Status: "PROVISIONING", - InstanceName: "skip-inst", + InstanceName: "audit-inst", Spec: map[string]any{"cpu": 1}, + AgentName: &agentName, } Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) Expect(dataStore.ServiceTypeInstance().MarkForDeletion(ctx, inst.ID)).To(Succeed()) scheduler.ProcessPendingDeletions(ctx) - // Instance should be marked as PENDING_PROVIDER, not deleted found, err := dataStore.ServiceTypeInstance().Get(ctx, inst.ID, true) Expect(err).NotTo(HaveOccurred()) - Expect(*found.DeletionStatus).To(Equal("PENDING_PROVIDER")) + Expect(*found.DeletionStatus).To(Equal("DELETED")) }) - It("skips instance when provider is unhealthy", func() { - Expect(db.Model(&model.Provider{}).Where("name = ?", providerName). - Update("health_status", model.HealthStatusUnhealthy).Error).NotTo(HaveOccurred()) - + It("marks DELETED when agent not registered", func() { + agentName := "unregistered-agent" inst := model.ServiceTypeInstance{ ID: uuid.New().String(), - ProviderName: providerName, + ServiceType: "vm", Status: "PROVISIONING", - InstanceName: "skip-unhealthy-inst", + InstanceName: "orphan-inst", Spec: map[string]any{"cpu": 1}, + AgentName: &agentName, } Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) Expect(dataStore.ServiceTypeInstance().MarkForDeletion(ctx, inst.ID)).To(Succeed()) @@ -125,18 +109,35 @@ var _ = Describe("Scheduler", func() { found, err := dataStore.ServiceTypeInstance().Get(ctx, inst.ID, true) Expect(err).NotTo(HaveOccurred()) - Expect(*found.DeletionStatus).To(Equal("PENDING_PROVIDER")) + Expect(*found.DeletionStatus).To(Equal("DELETED")) }) - It("proceeds with deletion when provider is ready", func() { - // Provider is already Ready from setup + It("marks DELETED for agent-routed instance regardless of service_type", func() { + agentName := "mismatch-agent" + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "storage", + Status: "PROVISIONING", + InstanceName: "mismatch-inst", + Spec: map[string]any{"size": 100}, + AgentName: &agentName, + } + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + Expect(dataStore.ServiceTypeInstance().MarkForDeletion(ctx, inst.ID)).To(Succeed()) + + scheduler.ProcessPendingDeletions(ctx) + + found, err := dataStore.ServiceTypeInstance().Get(ctx, inst.ID, true) + Expect(err).NotTo(HaveOccurred()) + Expect(*found.DeletionStatus).To(Equal("DELETED")) + }) - // Create an instance marked for deletion + It("treats instance without agent_name as agent-routed (agent-only world)", func() { inst := model.ServiceTypeInstance{ ID: uuid.New().String(), - ProviderName: providerName, + ServiceType: "vm", Status: "PROVISIONING", - InstanceName: "delete-inst", + InstanceName: "no-agent-inst", Spec: map[string]any{"cpu": 1}, } Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) @@ -144,9 +145,61 @@ var _ = Describe("Scheduler", func() { scheduler.ProcessPendingDeletions(ctx) - // Instance should be hard-deleted - _, err := dataStore.ServiceTypeInstance().Get(ctx, inst.ID, true) - Expect(err).To(HaveOccurred()) + found, err := dataStore.ServiceTypeInstance().Get(ctx, inst.ID, true) + Expect(err).NotTo(HaveOccurred()) + Expect(*found.DeletionStatus).To(Equal("DELETED")) + }) + }) + + Describe("Ack-driven deletion (agent + publisher configured)", func() { + It("stays SCHEDULED and does not mark DELETED after a successful publish, awaiting the agent's ack", func() { + agentName := "audit-agent" + pub := messaging.NewPublisher(&stubJetStream{}) + schedulerWithAgent := cleanup.NewScheduler(dataStore, pub, agentstore.NewAgent(db), &config.CleanupConfig{MaxRetries: 3}) + + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "deleting", + InstanceName: "ack-driven-inst", + Spec: map[string]any{"cpu": 1}, + AgentName: &agentName, + } + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + Expect(dataStore.ServiceTypeInstance().MarkForDeletion(ctx, inst.ID)).To(Succeed()) + + schedulerWithAgent.ProcessPendingDeletions(ctx) + + found, err := dataStore.ServiceTypeInstance().Get(ctx, inst.ID, true) + Expect(err).NotTo(HaveOccurred()) + Expect(*found.DeletionStatus).To(Equal("SCHEDULED")) + Expect(found.RetryCount).To(Equal(1)) + }) + + It("marks FAILED for manual intervention once retries are exhausted", func() { + agentName := "audit-agent" + pub := messaging.NewPublisher(&stubJetStream{}) + schedulerWithAgent := cleanup.NewScheduler(dataStore, pub, agentstore.NewAgent(db), &config.CleanupConfig{MaxRetries: 2}) + + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "deleting", + InstanceName: "exhausted-inst", + Spec: map[string]any{"cpu": 1}, + AgentName: &agentName, + RetryCount: 2, + } + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + Expect(dataStore.ServiceTypeInstance().MarkForDeletion(ctx, inst.ID)).To(Succeed()) + // MarkForDeletion resets retry_count; simulate prior attempts explicitly. + Expect(db.Model(&inst).Update("retry_count", 2).Error).NotTo(HaveOccurred()) + + schedulerWithAgent.ProcessPendingDeletions(ctx) + + found, err := dataStore.ServiceTypeInstance().Get(ctx, inst.ID, true) + Expect(err).NotTo(HaveOccurred()) + Expect(*found.DeletionStatus).To(Equal("FAILED")) }) }) }) diff --git a/internal/sp/config/config.go b/internal/sp/config/config.go index 29f0cff..daf6b4a 100644 --- a/internal/sp/config/config.go +++ b/internal/sp/config/config.go @@ -11,11 +11,10 @@ import ( ) type Config struct { - Database *DBConfig - Service *ServiceConfig - HealthCheck *HealthCheckConfig - NATS *NATSConfig - Cleanup *CleanupConfig + Database *DBConfig + Service *ServiceConfig + NATS *NATSConfig + Cleanup *CleanupConfig } type CleanupConfig struct { @@ -24,14 +23,6 @@ type CleanupConfig struct { Timeout time.Duration `envconfig:"CLEANUP_TIMEOUT" default:"10s"` } -type HealthCheckConfig struct { - Interval time.Duration `envconfig:"HEALTH_CHECK_INTERVAL" default:"10s"` - Timeout time.Duration `envconfig:"HEALTH_CHECK_TIMEOUT" default:"5s"` - MaxConsecutiveFailures int `envconfig:"HEALTH_CHECK_MAX_CONSECUTIVE_FAILURES" default:"3"` - BaseBackoffInterval time.Duration `envconfig:"HEALTH_CHECK_BASE_BACKOFF_INTERVAL" default:"10s"` - MaxBackoffInterval time.Duration `envconfig:"HEALTH_CHECK_MAX_BACKOFF_INTERVAL" default:"5m"` -} - type DBConfig struct { Type string `envconfig:"DB_TYPE" default:"pgsql"` Hostname string `envconfig:"DB_HOST" default:"localhost"` diff --git a/internal/sp/consumer/consumer.go b/internal/sp/consumer/consumer.go index 4ae7013..5823173 100644 --- a/internal/sp/consumer/consumer.go +++ b/internal/sp/consumer/consumer.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "log/slog" + "strings" "time" cloudevents "github.com/cloudevents/sdk-go/v2/event" @@ -179,7 +180,13 @@ func (c *StatusConsumer) handleMessage(ctx context.Context, msg jetstream.Msg) { return } - if err := c.store.ServiceTypeInstance().UpdateStatus(ctx, payload.Id, payload.Status, payload.Message); err != nil { + // This event's status string comes from an external CloudEvent producer + // we don't control, so it's normalized to the lowercase convention used + // throughout the rest of the status lifecycle before it crosses into our + // store, rather than trusting upstream casing (see internal/sp/store/model/status.go). + normalizedStatus := strings.ToLower(strings.TrimSpace(payload.Status)) + + if err := c.store.ServiceTypeInstance().UpdateStatus(ctx, payload.Id, normalizedStatus, payload.Message); err != nil { if errors.Is(err, rmstore.ErrInstanceNotFound) { slog.Warn("No instance found, skipping status update", "instance_id", payload.Id) _ = msg.Ack() @@ -190,6 +197,6 @@ func (c *StatusConsumer) handleMessage(ctx context.Context, msg jetstream.Msg) { return } - slog.Info("Instance status updated", "instance_id", payload.Id, "status", payload.Status) + slog.Info("Instance status updated", "instance_id", payload.Id, "status", normalizedStatus) _ = msg.Ack() } diff --git a/internal/sp/consumer/consumer_test.go b/internal/sp/consumer/consumer_test.go index 8db6538..4f6f88a 100644 --- a/internal/sp/consumer/consumer_test.go +++ b/internal/sp/consumer/consumer_test.go @@ -8,6 +8,7 @@ import ( "time" cloudevents "github.com/cloudevents/sdk-go/v2/event" + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" "github.com/dcm-project/control-plane/internal/sp/consumer" "github.com/dcm-project/control-plane/internal/sp/store" "github.com/dcm-project/control-plane/internal/sp/store/model" @@ -54,7 +55,7 @@ var _ = Describe("StatusConsumer", func() { sqlDB, err := db.DB() Expect(err).NotTo(HaveOccurred()) sqlDB.SetMaxOpenConns(1) - Expect(db.AutoMigrate(&model.ServiceTypeInstance{})).To(Succeed()) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) dataStore = store.NewStore(db) // Use the NATS test server URL from suite_test.go @@ -115,8 +116,7 @@ var _ = Describe("StatusConsumer", func() { createInstance := func(instanceID string) { instance := model.ServiceTypeInstance{ ID: instanceID, - ProviderName: "test-provider", - Status: "PROVISIONING", + Status: "provisioning", InstanceName: "test-instance", Spec: map[string]any{"cpu": "2"}, } @@ -134,7 +134,23 @@ var _ = Describe("StatusConsumer", func() { var inst model.ServiceTypeInstance db.Where("id = ?", instanceID).First(&inst) return inst.Status - }, 2*time.Second, 100*time.Millisecond).Should(Equal("RUNNING")) + }, 2*time.Second, 100*time.Millisecond).Should(Equal("running")) + }) + + It("normalizes mixed-case status casing from external producers to lowercase", func() { + // The external provider CloudEvent producer is not under our control and + // may send any casing; the DB's status column must stay consistently + // lowercase regardless (see internal/sp/store/model/status.go). + instanceID := uuid.New().String() + createInstance(instanceID) + + publishStatusEvent("kubevirt-sp", "vm", instanceID, "Running", "VM is running") + + Eventually(func() string { + var inst model.ServiceTypeInstance + db.Where("id = ?", instanceID).First(&inst) + return inst.Status + }, 2*time.Second, 100*time.Millisecond).Should(Equal("running")) }) It("updates status message along with status", func() { @@ -150,6 +166,19 @@ var _ = Describe("StatusConsumer", func() { }, 2*time.Second, 100*time.Millisecond).Should(Equal("VM crashed unexpectedly")) }) + It("stores the status from a FAILED event as lowercase", func() { + instanceID := uuid.New().String() + createInstance(instanceID) + + publishStatusEvent("kubevirt-sp", "vm", instanceID, "FAILED", "VM crashed unexpectedly") + + Eventually(func() string { + var inst model.ServiceTypeInstance + db.Where("id = ?", instanceID).First(&inst) + return inst.Status + }, 2*time.Second, 100*time.Millisecond).Should(Equal("failed")) + }) + It("handles events for non-existent instances gracefully", func() { publishStatusEvent("kubevirt-sp", "vm", "non-existent-id", "RUNNING", "VM is running") @@ -175,7 +204,7 @@ var _ = Describe("StatusConsumer", func() { var inst model.ServiceTypeInstance db.Where("id = ?", instanceID).First(&inst) return inst.Status - }, 2*time.Second, 100*time.Millisecond).Should(Equal("PROVISIONING")) + }, 2*time.Second, 100*time.Millisecond).Should(Equal("provisioning")) time.Sleep(200 * time.Millisecond) @@ -185,7 +214,7 @@ var _ = Describe("StatusConsumer", func() { var inst model.ServiceTypeInstance db.Where("id = ?", instanceID).First(&inst) return inst.Status - }, 2*time.Second, 100*time.Millisecond).Should(Equal("RUNNING")) + }, 2*time.Second, 100*time.Millisecond).Should(Equal("running")) }) It("Check succeeds while connected", func() { diff --git a/internal/sp/consumer/response_consumer.go b/internal/sp/consumer/response_consumer.go new file mode 100644 index 0000000..481328a --- /dev/null +++ b/internal/sp/consumer/response_consumer.go @@ -0,0 +1,366 @@ +package consumer + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "sync" + "time" + + agentstore "github.com/dcm-project/control-plane/internal/agent/store/agent" + "github.com/dcm-project/control-plane/internal/sp/messaging" + "github.com/dcm-project/control-plane/internal/sp/store" + "github.com/dcm-project/control-plane/internal/sp/store/model" + rmstore "github.com/dcm-project/control-plane/internal/sp/store/resource_manager" + "github.com/nats-io/nats.go/jetstream" +) + +const ( + consumerName = "control-plane-response-consumer" +) + +// defaultMaxDeliver/defaultAckWait are used when the caller passes <= 0, +// keeping existing callers (and tests) working without having to plumb +// config through everywhere. +const ( + defaultMaxDeliver = 10 + defaultAckWait = 30 * time.Second +) + +type cloudEvent struct { + Type string `json:"type"` + Data json.RawMessage `json:"data"` +} + +type eventData struct { + ResourceID string `json:"resource_id"` + // AgentName is checked against the instance's currently assigned + // agent_name before any status transition, rejecting late events from + // an agent superseded by self-healing. + AgentName string `json:"agent_name"` +} + +type ResponseConsumer struct { + js jetstream.JetStream + store store.Store + publisher *messaging.Publisher + agentStore agentstore.Agent + maxDeliver int + ackWait time.Duration + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup +} + +// NewResponseConsumer constructs a ResponseConsumer. maxDeliver bounds +// redeliveries of a message that keeps getting Nak'd; ackWait is how long +// JetStream waits for an ack before redelivering. Pass <= 0 for either to +// use the package defaults. +func NewResponseConsumer(js jetstream.JetStream, st store.Store, agentSt agentstore.Agent, maxDeliver int, ackWait time.Duration) *ResponseConsumer { + if maxDeliver <= 0 { + maxDeliver = defaultMaxDeliver + } + if ackWait <= 0 { + ackWait = defaultAckWait + } + return &ResponseConsumer{ + js: js, + store: st, + publisher: messaging.NewPublisher(js), + agentStore: agentSt, + maxDeliver: maxDeliver, + ackWait: ackWait, + } +} + +func (c *ResponseConsumer) Start(ctx context.Context) error { + c.ctx, c.cancel = context.WithCancel(ctx) + + // WorkQueuePolicy: this is a point-to-point work queue (one durable + // consumer, each message consumed and acked exactly once), so messages + // are removed once acked instead of retained forever. + stream, err := c.js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ + Name: messaging.ResponseStreamName, + Subjects: []string{messaging.ResponseSubject}, + Retention: jetstream.WorkQueuePolicy, + }) + if err != nil { + return fmt.Errorf("create response stream: %w", err) + } + + cons, err := stream.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{ + Durable: consumerName, + AckPolicy: jetstream.AckExplicitPolicy, + AckWait: c.ackWait, + MaxDeliver: c.maxDeliver, + }) + if err != nil { + return fmt.Errorf("create response consumer: %w", err) + } + + c.wg.Add(1) + go func() { + defer c.wg.Done() + c.consumeLoop(cons) + }() + return nil +} + +func (c *ResponseConsumer) Stop() { + if c.cancel != nil { + c.cancel() + } + c.wg.Wait() +} + +// consumeLoop uses only c.ctx for shutdown, so Stop() cancelling it is +// always sufficient to exit the loop, even if a caller cancels the ctx +// passed to Start() directly. +func (c *ResponseConsumer) consumeLoop(cons jetstream.Consumer) { + for { + select { + case <-c.ctx.Done(): + return + default: + } + + msgs, err := cons.Fetch(1, jetstream.FetchMaxWait(500*time.Millisecond)) + if err != nil { + select { + case <-c.ctx.Done(): + return + case <-time.After(time.Second): + } + continue + } + for msg := range msgs.Messages() { + c.handleMessage(msg) + } + } +} + +func (c *ResponseConsumer) handleMessage(msg jetstream.Msg) { + var ce cloudEvent + if err := json.Unmarshal(msg.Data(), &ce); err != nil { + slog.Error("malformed cloud event, acking to discard", "error", err) + _ = msg.Ack() + return + } + + var data eventData + if err := json.Unmarshal(ce.Data, &data); err != nil { + slog.Error("malformed event data, acking to discard", "error", err) + _ = msg.Ack() + return + } + + if data.ResourceID == "" { + slog.Error("event missing resource_id, acking to discard") + _ = msg.Ack() + return + } + + // agent_name is required on every response CE payload; never fall back + // to "trust it anyway" for a missing value. + if data.AgentName == "" { + slog.Error("event missing agent_name, acking to discard", "resource_id", data.ResourceID) + _ = msg.Ack() + return + } + + ctx := c.ctx + + switch ce.Type { + case messaging.CETypeCancelRejected: + c.handleCancelRejected(ctx, data, msg) + return + case messaging.CETypeDeletionAcknowledged: + c.handleDeletionAcknowledged(ctx, data, msg) + return + case messaging.CETypeRequestQueued: + c.handleRequestQueued(ctx, data, msg) + return + default: + } + + // fromStatuses CAS-guards each transition in addition to (not instead + // of) UpdateStatusFrom's agent_name identity check below: status alone + // wouldn't catch a late ack from a superseded agent if the instance's + // status has since cycled back into an allowed fromStatus under a new one. + var newStatus string + var fromStatuses []string + switch ce.Type { + case messaging.CETypeCreationAcknowledged: + newStatus = model.StatusProvisioning + fromStatuses = []string{model.StatusPending, model.StatusQueued} + case messaging.CETypeError: + newStatus = model.StatusFailed + fromStatuses = []string{model.StatusPending, model.StatusQueued, model.StatusProvisioning} + case messaging.CETypeCancelAcknowledged: + newStatus = model.StatusCancelled + fromStatuses = []string{model.StatusQueued} + default: + slog.Warn("unknown event type, acking", "type", ce.Type) + _ = msg.Ack() + return + } + + stiStore := c.store.ServiceTypeInstance() + applied, err := stiStore.UpdateStatusFrom(ctx, data.ResourceID, fromStatuses, data.AgentName, newStatus, "") + if err != nil { + slog.Error("failed to update status, nacking", "resource_id", data.ResourceID, "error", err) + _ = msg.NakWithDelay(5 * time.Second) + return + } + if !applied { + // A second read would tell us status- vs agent-mismatch but + // reintroduce a TOCTOU window purely for logging; include agent_name + // so operators can cross-reference it against the DB instead. + slog.Info("stale or duplicate status event, instance already moved on, agent mismatch, or not found, acking", + "resource_id", data.ResourceID, "event_type", ce.Type, "agent_name", data.AgentName) + } + + _ = msg.Ack() +} + +// handleRequestQueued transitions status to "queued" and resets +// pending_started_at, so the queued-timeout sweep measures from the moment +// the agent queued the request rather than from the original pending +// timestamp (a long-pending instance would otherwise be cancelled +// immediately upon queueing). +func (c *ResponseConsumer) handleRequestQueued(ctx context.Context, data eventData, msg jetstream.Msg) { + if err := c.store.ServiceTypeInstance().MarkQueued(ctx, data.ResourceID, data.AgentName); err != nil { + if errors.Is(err, rmstore.ErrInstanceNotFound) { + slog.Warn("instance not found, status mismatch, or agent mismatch for request-queued, acking to discard poison message", + "resource_id", data.ResourceID, "agent_name", data.AgentName) + _ = msg.Ack() + return + } + slog.Error("failed to mark instance queued, nacking", "resource_id", data.ResourceID, "error", err) + _ = msg.NakWithDelay(5 * time.Second) + return + } + _ = msg.Ack() +} + +// handleDeletionAcknowledged finalizes a delete once the agent confirms the +// physical resource is gone. It branches on deletion_status/status because +// the same event serves both delete paths: non-deferred deletes are +// hard-deleted now (nothing else is keeping them around); deferred deletes +// are soft-completed to keep their tombstone. Any other combination is a +// late/duplicate redelivery and a no-op, so it can't erase an +// already-finalized tombstone. +func (c *ResponseConsumer) handleDeletionAcknowledged(ctx context.Context, data eventData, msg jetstream.Msg) { + stiStore := c.store.ServiceTypeInstance() + + instance, err := stiStore.Get(ctx, data.ResourceID, true) + if err != nil { + if errors.Is(err, rmstore.ErrInstanceNotFound) { + slog.Info("deletion-acknowledged: instance already gone, acking", "resource_id", data.ResourceID) + _ = msg.Ack() + return + } + slog.Error("deletion-acknowledged: failed to look up instance, nacking", "resource_id", data.ResourceID, "error", err) + _ = msg.NakWithDelay(5 * time.Second) + return + } + + switch { + case instance.Status == model.StatusDeleting: + // Checked ahead of DeletionStatus: a non-deferred delete must + // always be fully removed once acknowledged, regardless of whether + // its best-effort MarkForDeletion enrollment also set SCHEDULED. + if err := stiStore.HardDeleteFromAgent(ctx, data.ResourceID, data.AgentName); err != nil { + if !errors.Is(err, rmstore.ErrInstanceNotFound) { + slog.Error("deletion-acknowledged: failed to hard-delete instance, nacking", "resource_id", data.ResourceID, "error", err) + _ = msg.NakWithDelay(5 * time.Second) + return + } + slog.Info("deletion-acknowledged: instance already gone or agent mismatch, acking", + "resource_id", data.ResourceID, "agent_name", data.AgentName) + } + case instance.Status == model.StatusPendingDeletion, + instance.DeletionStatus != nil && *instance.DeletionStatus == rmstore.DeletionStatusScheduled: + // Status=pending_deletion is matched even without deletion_status + // set, so a cancel-rejected retry whose own MarkForDeletion + // enrollment failed doesn't fall through to the default case and + // get stranded as "stale". + if err := stiStore.MarkDeletionCompleteFromAgent(ctx, data.ResourceID, data.AgentName); err != nil { + if errors.Is(err, rmstore.ErrInstanceNotFound) { + slog.Info("deletion-acknowledged: instance already gone or agent mismatch, acking", + "resource_id", data.ResourceID, "agent_name", data.AgentName) + } else { + slog.Error("deletion-acknowledged: failed to mark deferred deletion complete, nacking", "resource_id", data.ResourceID, "error", err) + _ = msg.NakWithDelay(5 * time.Second) + return + } + } + default: + slog.Info("deletion-acknowledged: deletion already finalized or instance was never deleting, ignoring stale/duplicate ack", + "resource_id", data.ResourceID, "status", instance.Status, "deletion_status", instance.DeletionStatus) + } + _ = msg.Ack() +} + +// handleCancelRejected transitions a queued/cancelled instance back to +// "pending_deletion" so the delete can be retried, then re-publishes the +// delete. cancellableStatuses is deliberately an ALLOW-list of just +// {queued, cancelled} - the only statuses a genuine ack of +// sweep.cancelQueuedInstance's cancel request can arrive during - not a +// broader "any non-terminal status": a wider list could match a LATER, +// unrelated reassignment cycle and delete a freshly re-provisioned instance +// out from under its new agent. +func (c *ResponseConsumer) handleCancelRejected(ctx context.Context, data eventData, msg jetstream.Msg) { + stiStore := c.store.ServiceTypeInstance() + + cancellableStatuses := []string{model.StatusQueued, model.StatusCancelled} + applied, err := stiStore.UpdateStatusFrom(ctx, data.ResourceID, cancellableStatuses, data.AgentName, model.StatusPendingDeletion, "") + if err != nil { + slog.Error("cancel-rejected: failed to update status, nacking", "resource_id", data.ResourceID, "error", err) + _ = msg.NakWithDelay(5 * time.Second) + return + } + if !applied { + slog.Info("cancel-rejected: instance already moved to a terminal state or agent mismatch, skipping redundant delete", + "resource_id", data.ResourceID, "agent_name", data.AgentName) + _ = msg.Ack() + return + } + + // Enroll in cleanup's retry/timeout tracking, not just the best-effort + // republish below: otherwise a failed republish is never retried. + if err := stiStore.MarkForDeletion(ctx, data.ResourceID); err != nil { + slog.Error("cancel-rejected: failed to enroll instance in deletion retry tracking", "resource_id", data.ResourceID, "error", err) + } + + instance, err := stiStore.Get(ctx, data.ResourceID, true) + if err == nil && instance.AgentName != nil { + subject, ok := c.resolveAgentTopic(ctx, *instance.AgentName) + if ok { + payload := messaging.DeletePayload{ + ResourceID: data.ResourceID, + ServiceType: instance.ServiceType, + } + if pubErr := c.publisher.PublishDelete(ctx, subject, payload); pubErr != nil { + slog.Warn("cancel-rejected: publish delete failed, sweep will retry", "resource_id", data.ResourceID, "error", pubErr) + } + } + } + + _ = msg.Ack() +} + +func (c *ResponseConsumer) resolveAgentTopic(ctx context.Context, agentName string) (string, bool) { + if c.agentStore == nil { + slog.Error("resolveAgentTopic: agent store not configured, cannot resolve topic", "agent_name", agentName) + return "", false + } + agent, err := c.agentStore.GetByName(ctx, agentName) + if err != nil { + slog.Warn("resolveAgentTopic: agent not found, skipping publish", "agent_name", agentName, "error", err) + return "", false + } + return agent.TopicName, true +} diff --git a/internal/sp/consumer/response_consumer_test.go b/internal/sp/consumer/response_consumer_test.go new file mode 100644 index 0000000..5d8e4c7 --- /dev/null +++ b/internal/sp/consumer/response_consumer_test.go @@ -0,0 +1,637 @@ +package consumer_test + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "time" + + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" + "github.com/dcm-project/control-plane/internal/sp/consumer" + "github.com/dcm-project/control-plane/internal/sp/store" + "github.com/dcm-project/control-plane/internal/sp/store/model" + "github.com/google/uuid" + natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +var _ = Describe("ResponseConsumer", func() { + var ( + ns *natsserver.Server + nc *nats.Conn + js jetstream.JetStream + dataStore store.Store + db *gorm.DB + rc *consumer.ResponseConsumer + ctx context.Context + ) + + BeforeEach(func() { + opts := &natsserver.Options{ + Host: "127.0.0.1", + Port: -1, + JetStream: true, + StoreDir: GinkgoT().TempDir(), + } + var err error + ns, err = natsserver.NewServer(opts) + Expect(err).NotTo(HaveOccurred()) + ns.Start() + Expect(ns.ReadyForConnections(2 * time.Second)).To(BeTrue()) + + nc, err = nats.Connect(ns.ClientURL()) + Expect(err).NotTo(HaveOccurred()) + + js, err = jetstream.New(nc) + Expect(err).NotTo(HaveOccurred()) + + db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + Expect(err).NotTo(HaveOccurred()) + // sqlite's ":memory:" DSN gives each new physical connection its own + // empty database, so once the consumer's background goroutine and the + // test's own Eventually assertions query concurrently, a second + // pooled connection would see "no such table" instead of the + // migrated schema. + sqlDB, err := db.DB() + Expect(err).NotTo(HaveOccurred()) + sqlDB.SetMaxOpenConns(1) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) + Expect(db.Create(&agentmodel.Agent{ID: uuid.New().String(), Name: testAgentName, TopicName: "dcm.agent.test-agent"}).Error).NotTo(HaveOccurred()) + + dataStore = store.NewStore(db) + rc = consumer.NewResponseConsumer(js, dataStore, nil, 0, 0) + ctx = context.Background() + }) + + AfterEach(func() { + rc.Stop() + if nc != nil { + nc.Close() + } + if ns != nil { + ns.Shutdown() + } + sqlDB, _ := db.DB() + _ = sqlDB.Close() + }) + + It("stops promptly once the parent context is cancelled directly, without a separate stopCh", func() { + // consumeLoop's select only watches c.ctx.Done(), so Stop() + // cancelling that same ctx is the single canonical shutdown signal. + startCtx, cancel := context.WithCancel(context.Background()) + Expect(rc.Start(startCtx)).To(Succeed()) + + cancel() + + stopped := make(chan struct{}) + go func() { + rc.Stop() + close(stopped) + }() + // consumeLoop's Fetch call waits up to 500ms and its error-retry + // path waits up to 1s, so 2s comfortably bounds a healthy exit + // while still catching a hang if ctx.Done() were ignored. + Eventually(stopped, 2*time.Second).Should(BeClosed()) + }) + + It("configures the response stream and consumer with WorkQueuePolicy, MaxDeliver and AckWait", func() { + rc2 := consumer.NewResponseConsumer(js, dataStore, nil, 5, 15*time.Second) + Expect(rc2.Start(ctx)).To(Succeed()) + defer rc2.Stop() + + stream, err := js.Stream(ctx, "dcm-agent-responses") + Expect(err).NotTo(HaveOccurred()) + info, err := stream.Info(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Config.Retention).To(Equal(jetstream.WorkQueuePolicy)) + + cons, err := stream.Consumer(ctx, "control-plane-response-consumer") + Expect(err).NotTo(HaveOccurred()) + consInfo, err := cons.Info(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(consInfo.Config.MaxDeliver).To(Equal(5)) + Expect(consInfo.Config.AckWait).To(Equal(15 * time.Second)) + }) + + It("transitions PENDING to PROVISIONING on creation-acknowledged", func() { + instance := createPendingInstance(ctx, db) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.creation-acknowledged", instance.ID, testAgentName) + + Eventually(func() string { + return currentStatus(db, instance.ID) + }, 2*time.Second, 20*time.Millisecond).Should(Equal("provisioning")) + }) + + // A late creation-acknowledged from a superseded agent must not apply + // even though the instance is still "pending" - a status a genuine ack + // from the new agent could also legitimately arrive during. + It("ignores a creation-acknowledged from a superseded agent even though status still matches (identity check)", func() { + instance := createPendingInstance(ctx, db) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.creation-acknowledged", instance.ID, staleAgentName) + + Consistently(func() string { + return currentStatus(db, instance.ID) + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("pending")) + }) + + // A mismatch is logged with agent_name included, so operators can + // cross-reference "stale agent" against "instance already moved on". + It("includes agent_name in the log line when a creation-acknowledged is rejected for an agent mismatch", func() { + instance := createPendingInstance(ctx, db) + + var buf bytes.Buffer + prevLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + defer slog.SetDefault(prevLogger) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.creation-acknowledged", instance.ID, staleAgentName) + + Eventually(buf.String, 2*time.Second, 20*time.Millisecond).Should(SatisfyAll( + ContainSubstring("stale or duplicate status event"), + ContainSubstring("agent_name="+staleAgentName), + )) + }) + + It("transitions to FAILED on error event", func() { + instance := createPendingInstance(ctx, db) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.error", instance.ID, testAgentName) + + Eventually(func() string { + return currentStatus(db, instance.ID) + }, 2*time.Second, 20*time.Millisecond).Should(Equal("failed")) + }) + + // Same mismatch treatment as creation-acknowledged, for the error event. + It("ignores an error event from a superseded agent even though status still matches (identity check)", func() { + instance := createPendingInstance(ctx, db) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.error", instance.ID, staleAgentName) + + Consistently(func() string { + return currentStatus(db, instance.ID) + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("pending")) + }) + + It("transitions to QUEUED and resets the pending timer on request-queued", func() { + instance := createPendingInstance(ctx, db) + // Simulate the instance having been pending for a while already, so a + // stale pending_started_at would make the queued-timeout sweep see it + // as immediately overdue. + staleTime := time.Now().Add(-1 * time.Hour) + Expect(db.Model(&instance).Update("pending_started_at", staleTime).Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.request-queued", instance.ID, testAgentName) + + Eventually(func() string { + return currentStatus(db, instance.ID) + }, 2*time.Second, 20*time.Millisecond).Should(Equal("queued")) + + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", instance.ID).Error).NotTo(HaveOccurred()) + Expect(updated.PendingStartedAt).NotTo(BeNil()) + Expect(*updated.PendingStartedAt).To(BeTemporally(">", staleTime)) + }) + + // A stale request-queued from a superseded agent must not mark the + // instance queued or reset its pending timer. + It("ignores a request-queued from a superseded agent even though status still matches (identity check)", func() { + instance := createPendingInstance(ctx, db) + staleTime := time.Now().Add(-1 * time.Hour) + Expect(db.Model(&instance).Update("pending_started_at", staleTime).Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.request-queued", instance.ID, staleAgentName) + + Consistently(func() string { + return currentStatus(db, instance.ID) + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("pending")) + + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", instance.ID).Error).NotTo(HaveOccurred()) + Expect(updated.PendingStartedAt).NotTo(BeNil()) + Expect(*updated.PendingStartedAt).To(BeTemporally("==", staleTime)) + }) + + It("hard-deletes a non-deferred instance on deletion-acknowledged", func() { + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("status", "deleting").Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.deletion-acknowledged", instance.ID, testAgentName) + + Eventually(func() error { + return db.First(&model.ServiceTypeInstance{}, "id = ?", instance.ID).Error + }, 2*time.Second, 100*time.Millisecond).Should(MatchError(gorm.ErrRecordNotFound)) + }) + + // A stale deletion-acknowledged from a superseded agent must not + // hard-delete the row, for the non-deferred branch. + It("ignores a deletion-acknowledged (non-deferred branch) from a superseded agent", func() { + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("status", "deleting").Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.deletion-acknowledged", instance.ID, staleAgentName) + + Consistently(func() error { + return db.First(&model.ServiceTypeInstance{}, "id = ?", instance.ID).Error + }, 300*time.Millisecond, 20*time.Millisecond).Should(Succeed()) + }) + + It("marks a deferred instance DELETED (soft) on deletion-acknowledged", func() { + // A deferred DeleteInstance never touches Status (only + // deletion_status) - "deleting" is exclusively the non-deferred + // marker, so this leaves Status at its original "pending" to match + // production reality. + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("deletion_status", "SCHEDULED").Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.deletion-acknowledged", instance.ID, testAgentName) + + Eventually(func() string { + var updated model.ServiceTypeInstance + if err := db.First(&updated, "id = ?", instance.ID).Error; err != nil { + return "" + } + if updated.DeletionStatus == nil { + return "" + } + return *updated.DeletionStatus + }, 2*time.Second, 100*time.Millisecond).Should(Equal("DELETED")) + + // The row must still exist (soft-delete tombstone), unlike the + // non-deferred case above. + Expect(db.First(&model.ServiceTypeInstance{}, "id = ?", instance.ID).Error).NotTo(HaveOccurred()) + }) + + // Same mismatch treatment, for the deferred (soft-complete) branch. + It("ignores a deletion-acknowledged (deferred branch) from a superseded agent", func() { + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("deletion_status", "SCHEDULED").Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.deletion-acknowledged", instance.ID, staleAgentName) + + Consistently(func() string { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", instance.ID).Error).NotTo(HaveOccurred()) + if updated.DeletionStatus == nil { + return "" + } + return *updated.DeletionStatus + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("SCHEDULED")) + }) + + It("finalizes a pending_deletion instance on deletion-acknowledged even if its MarkForDeletion enrollment failed (R2 S1/S2: finding #2/#3)", func() { + // handleCancelRejected best-effort calls MarkForDeletion after + // transitioning to pending_deletion; if that call fails, + // deletion_status stays nil while Status is pending_deletion. The + // switch must still key off Status==pending_deletion in that case - + // otherwise this ack falls through to the default "stale" branch and + // strands the instance in pending_deletion forever. + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("status", model.StatusPendingDeletion).Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.deletion-acknowledged", instance.ID, testAgentName) + + Eventually(func() string { + var updated model.ServiceTypeInstance + if err := db.First(&updated, "id = ?", instance.ID).Error; err != nil { + return "" + } + if updated.DeletionStatus == nil { + return "" + } + return *updated.DeletionStatus + }, 2*time.Second, 100*time.Millisecond).Should(Equal("DELETED")) + + Expect(db.First(&model.ServiceTypeInstance{}, "id = ?", instance.ID).Error).NotTo(HaveOccurred()) + }) + + It("ignores a late/duplicate deletion-acknowledged instead of erasing an existing DELETED tombstone (A)", func() { + // A DELETED tombstone only ever arises via the deferred path (a + // non-deferred delete is hard-deleted, not soft-marked), which never + // sets Status to "deleting" - leave it at "pending" to match. + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("deletion_status", "DELETED").Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.deletion-acknowledged", instance.ID, testAgentName) + + // Asserting a no-op: Consistently (not a fixed sleep-then-assert) + // gives the consumer the same window to WRONGLY erase the tombstone + // while actively re-checking throughout it, rather than a single + // read after one fixed delay. + Consistently(func() string { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", instance.ID).Error).NotTo(HaveOccurred()) + if updated.DeletionStatus == nil { + return "" + } + return *updated.DeletionStatus + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("DELETED")) + }) + + It("ignores a late/duplicate deletion-acknowledged instead of erasing an existing FAILED audit record (A)", func() { + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("deletion_status", "FAILED").Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.deletion-acknowledged", instance.ID, testAgentName) + + Consistently(func() string { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", instance.ID).Error).NotTo(HaveOccurred()) + if updated.DeletionStatus == nil { + return "" + } + return *updated.DeletionStatus + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("FAILED")) + }) + + It("publishes deletion on cancel-rejected and enrolls it in cleanup retry tracking", func() { + // cancel-rejected is only ever a genuine ack of the sweep's + // queued-timeout PublishCancel, which fires exclusively from + // "queued" - so that (or the "cancelled" it transitions to) is the + // only realistic precondition, not "pending". + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("status", model.StatusQueued).Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.cancel-rejected", instance.ID, testAgentName) + + Eventually(func() string { + return currentStatus(db, instance.ID) + }, 2*time.Second, 20*time.Millisecond).Should(Equal("pending_deletion")) + + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", instance.ID).Error).NotTo(HaveOccurred()) + // D: enrolled via MarkForDeletion so the cleanup scheduler will + // retry/time-out/audit-giveup this delete even if the immediate + // republish above fails or the ack never arrives. + Expect(updated.DeletionStatus).NotTo(BeNil()) + Expect(*updated.DeletionStatus).To(Equal("SCHEDULED")) + }) + + It("does not clobber a terminal state on cancel-rejected (CAS guard)", func() { + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("status", "failed").Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.cancel-rejected", instance.ID, testAgentName) + + Consistently(func() string { + return currentStatus(db, instance.ID) + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("failed")) + }) + + It("ignores a late cancel-rejected for an instance the self-healing loop already reassigned (R2 S3: finding #1)", func() { + // Instance has been reassigned to a new agent (status "pending") by + // the time a cancel-rejected for the original request arrives late; + // status alone (not agent identity) must already reject it, so + // agent_name is kept matching to isolate that guard. + instance := createPendingInstance(ctx, db) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.cancel-rejected", instance.ID, testAgentName) + + Consistently(func() string { + return currentStatus(db, instance.ID) + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("pending")) + + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", instance.ID).Error).NotTo(HaveOccurred()) + Expect(updated.DeletionStatus).To(BeNil()) + }) + + // A second, independent guard, this time closed by identity rather than + // status: even while the instance is still in an allowed + // cancellableStatus, an agent_name mismatch must reject the event. + It("ignores a cancel-rejected from a superseded agent even though status still matches (identity check)", func() { + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("status", model.StatusQueued).Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.cancel-rejected", instance.ID, staleAgentName) + + Consistently(func() string { + return currentStatus(db, instance.ID) + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("queued")) + + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", instance.ID).Error).NotTo(HaveOccurred()) + Expect(updated.DeletionStatus).To(BeNil()) + }) + + It("NakWithDelay on transient failures", func() { + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.creation-acknowledged", "nonexistent-id", testAgentName) + time.Sleep(500 * time.Millisecond) + }) + + It("Ack + log on permanent failures (malformed)", func() { + Expect(rc.Start(ctx)).To(Succeed()) + + ctx := context.Background() + _, err := js.Publish(ctx, "dcm.agents.responses", []byte("not-json")) + Expect(err).NotTo(HaveOccurred()) + time.Sleep(500 * time.Millisecond) + }) + + // A malformed event with no agent_name is acked and discarded without + // touching the instance, same as an empty/missing resource_id. + It("acks and discards an event missing agent_name without touching the instance", func() { + instance := createPendingInstance(ctx, db) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEventNoAgentName(js, "dcm.agent.creation-acknowledged", instance.ID) + + Consistently(func() string { + return currentStatus(db, instance.ID) + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("pending")) + }) + + // The blank-check in handleMessage must reject a missing agent_name on + // its own merits, not merely benefit from the store's WHERE clause never + // matching blank against a non-blank stored value - so the instance's + // own agent_name is blank here too. + It("acks and discards an event missing agent_name even when the instance's own agent_name is also blank", func() { + blank := "" + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("agent_name", blank).Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEventNoAgentName(js, "dcm.agent.creation-acknowledged", instance.ID) + + Consistently(func() string { + return currentStatus(db, instance.ID) + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("pending")) + }) + + It("transitions QUEUED to CANCELLED on cancel-acknowledged", func() { + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("status", model.StatusQueued).Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.cancel-acknowledged", instance.ID, testAgentName) + + Eventually(func() string { + return currentStatus(db, instance.ID) + }, 2*time.Second, 20*time.Millisecond).Should(Equal("cancelled")) + }) + + It("ignores a stale cancel-acknowledged for an instance the self-healing loop already reassigned (F: split-brain)", func() { + // Simulate: the queued-timeout sweep already cancelled+reassigned + // this instance to a new agent (status is back to "pending"), and + // the OLD agent's cancel-acknowledged for the original cancel + // request arrives late. It must not clobber the fresh "pending" + // state set up for the new agent. Status alone already guards this + // (pending is not in fromStatuses={queued}) - kept as testAgentName + // to isolate the status-CAS behavior from the identity check + // exercised below. + instance := createPendingInstance(ctx, db) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.cancel-acknowledged", instance.ID, testAgentName) + + Consistently(func() string { + return currentStatus(db, instance.ID) + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("pending")) + }) + + // Same mismatch treatment as creation-acknowledged/error, for + // cancel-acknowledged. + It("ignores a cancel-acknowledged from a superseded agent even though status still matches (identity check)", func() { + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("status", model.StatusQueued).Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.cancel-acknowledged", instance.ID, staleAgentName) + + Consistently(func() string { + return currentStatus(db, instance.ID) + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("queued")) + }) + + It("ignores a stale error event for an instance that already moved past provisioning", func() { + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("status", model.StatusCancelled).Error).NotTo(HaveOccurred()) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.error", instance.ID, testAgentName) + + Consistently(func() string { + return currentStatus(db, instance.ID) + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal("cancelled")) + }) +}) + +// currentStatus polls an instance's current status for use with +// Eventually/Consistently, instead of a fixed time.Sleep before one read. +func currentStatus(db *gorm.DB, id string) string { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", id).Error).NotTo(HaveOccurred()) + return updated.Status +} + +func createPendingInstance(_ context.Context, db *gorm.DB) model.ServiceTypeInstance { + agentName := testAgentName + now := time.Now() + instance := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "pending", + InstanceName: "test-instance", + Spec: map[string]any{"cpu": 4}, + AgentName: &agentName, + PendingStartedAt: &now, + } + Expect(db.Create(&instance).Error).NotTo(HaveOccurred()) + return instance +} + +// testAgentName is the agent_name used by createPendingInstance's fixture +// and by publishAgentEvent's default matching-agent call sites, so a +// "genuine ack" scenario is the default and mismatch scenarios explicitly +// opt into a different, stale agent name. +const testAgentName = "test-agent" + +// staleAgentName simulates a superseded agent's late event: an agent that +// no longer owns the instance (self-healing has since reassigned it to +// testAgentName) sending a delayed ack for its original assignment. +const staleAgentName = "stale-agent" + +func publishAgentEvent(js jetstream.JetStream, eventType string, resourceID string, agentName string) { + data, err := json.Marshal(map[string]any{ + "specversion": "1.0", + "type": eventType, + "source": "test", + "id": uuid.New().String(), + "data": map[string]any{"resource_id": resourceID, "agent_name": agentName}, + }) + Expect(err).NotTo(HaveOccurred()) + ctx := context.Background() + _, err = js.Publish(ctx, "dcm.agents.responses", data) + Expect(err).NotTo(HaveOccurred()) +} + +// publishAgentEventNoAgentName publishes a response event with resource_id +// but no agent_name field at all. +func publishAgentEventNoAgentName(js jetstream.JetStream, eventType string, resourceID string) { + data, err := json.Marshal(map[string]any{ + "specversion": "1.0", + "type": eventType, + "source": "test", + "id": uuid.New().String(), + "data": map[string]any{"resource_id": resourceID}, + }) + Expect(err).NotTo(HaveOccurred()) + ctx := context.Background() + _, err = js.Publish(ctx, "dcm.agents.responses", data) + Expect(err).NotTo(HaveOccurred()) +} diff --git a/internal/sp/handlers/provider/handler.go b/internal/sp/handlers/provider/handler.go deleted file mode 100644 index f29f89e..0000000 --- a/internal/sp/handlers/provider/handler.go +++ /dev/null @@ -1,170 +0,0 @@ -// Package provider implements HTTP handlers for the Provider API. -package provider - -import ( - "context" - - providerserver "github.com/dcm-project/control-plane/internal/sp/api/provider" - "github.com/dcm-project/control-plane/internal/sp/logging" - "github.com/dcm-project/control-plane/internal/sp/service" - providersvc "github.com/dcm-project/control-plane/internal/sp/service/provider" -) - -// Handler implements the generated StrictServerInterface for the Provider API. -type Handler struct { - providerService *providersvc.ProviderService -} - -// NewHandler creates a new Handler with the given provider service. -func NewHandler(providerService *providersvc.ProviderService) *Handler { - return &Handler{providerService: providerService} -} - -// Ensure Handler implements StrictServerInterface -var _ providerserver.StrictServerInterface = (*Handler)(nil) - -func (h *Handler) ListProviders(ctx context.Context, request providerserver.ListProvidersRequestObject) (providerserver.ListProvidersResponseObject, error) { - log := logging.FromContext(ctx) - log.Debug("ListProviders request received", - "type", request.Params.Type, - "page_size", request.Params.MaxPageSize, - ) - - var serviceType string - var maxPageSize int - var pageToken string - - if request.Params.Type != nil { - serviceType = *request.Params.Type - } - if request.Params.MaxPageSize != nil { - maxPageSize = *request.Params.MaxPageSize - } - if request.Params.PageToken != nil { - pageToken = *request.Params.PageToken - } - - result, err := h.providerService.ListProviders(ctx, serviceType, maxPageSize, pageToken) - if err != nil { - logServiceError(ctx, "ListProviders failed", err) - if svcErr, ok := err.(*service.ServiceError); ok && svcErr.Code == service.ErrCodeValidation { - return providerserver.ListProviders400ApplicationProblemPlusJSONResponse(newError("validation-error", "Invalid request", svcErr.Message, 400)), nil - } - return providerserver.ListProviders400ApplicationProblemPlusJSONResponse(newError("list-error", "Failed to list providers", err.Error(), 400)), nil - } - - response := providerserver.ListProviders200JSONResponse{Providers: &result.Providers} - if result.NextPageToken != "" { - response.NextPageToken = &result.NextPageToken - } - - log.Debug("ListProviders completed", "count", len(result.Providers)) - return response, nil -} - -func (h *Handler) CreateProvider(ctx context.Context, request providerserver.CreateProviderRequestObject) (providerserver.CreateProviderResponseObject, error) { - log := logging.FromContext(ctx) - log.Debug("CreateProvider request received", - "client_id", request.Params.Id, - "name", request.Body.Name, - ) - - response, updated, err := h.providerService.RegisterOrUpdateProvider(ctx, request.Body, request.Params.Id) - if err != nil { - logServiceError(ctx, "CreateProvider failed", err) - if svcErr, ok := err.(*service.ServiceError); ok { - switch svcErr.Code { - case service.ErrCodeValidation: - return providerserver.CreateProvider400ApplicationProblemPlusJSONResponse(newError("validation-error", "Validation failed", svcErr.Message, 400)), nil - case service.ErrCodeConflict: - return providerserver.CreateProvider409ApplicationProblemPlusJSONResponse(newError("conflict", "Resource conflict", svcErr.Message, 409)), nil - } - } - return providerserver.CreateProvider400ApplicationProblemPlusJSONResponse(newError("create-error", "Failed to create provider", err.Error(), 400)), nil - } - - if updated { - log.Info("Provider updated", "provider_id", *response.Id) - return providerserver.CreateProvider200JSONResponse(*response), nil - } - log.Info("Provider created", "provider_id", *response.Id) - return providerserver.CreateProvider201JSONResponse(*response), nil -} - -func (h *Handler) GetProvider(ctx context.Context, request providerserver.GetProviderRequestObject) (providerserver.GetProviderResponseObject, error) { - log := logging.FromContext(ctx) - log.Debug("GetProvider request received", "provider_id", request.ProviderId) - - provider, err := h.providerService.GetProvider(ctx, request.ProviderId) - if err != nil { - logServiceError(ctx, "GetProvider failed", err, "provider_id", request.ProviderId) - if svcErr, ok := err.(*service.ServiceError); ok && svcErr.Code == service.ErrCodeNotFound { - return providerserver.GetProvider404ApplicationProblemPlusJSONResponse(newError("not-found", "Provider not found", svcErr.Message, 404)), nil - } - return providerserver.GetProvider400ApplicationProblemPlusJSONResponse(newError("get-error", "Failed to get provider", err.Error(), 400)), nil - } - - log.Debug("GetProvider completed", "provider_id", request.ProviderId) - return providerserver.GetProvider200JSONResponse(*provider), nil -} - -func (h *Handler) ApplyProvider(ctx context.Context, request providerserver.ApplyProviderRequestObject) (providerserver.ApplyProviderResponseObject, error) { - log := logging.FromContext(ctx) - log.Debug("ApplyProvider request received", "provider_id", request.ProviderId) - - provider, err := h.providerService.UpdateProvider(ctx, request.ProviderId, request.Body) - if err != nil { - logServiceError(ctx, "ApplyProvider failed", err, "provider_id", request.ProviderId) - if svcErr, ok := err.(*service.ServiceError); ok { - switch svcErr.Code { - case service.ErrCodeNotFound: - return providerserver.ApplyProvider404ApplicationProblemPlusJSONResponse(newError("not-found", "Provider not found", svcErr.Message, 404)), nil - case service.ErrCodeConflict: - return providerserver.ApplyProvider409ApplicationProblemPlusJSONResponse(newError("conflict", "Name conflict", svcErr.Message, 409)), nil - } - } - return providerserver.ApplyProvider400ApplicationProblemPlusJSONResponse(newError("update-error", "Failed to update provider", err.Error(), 400)), nil - } - - log.Info("Provider updated", "provider_id", request.ProviderId) - return providerserver.ApplyProvider200JSONResponse(*provider), nil -} - -func (h *Handler) DeleteProvider(ctx context.Context, request providerserver.DeleteProviderRequestObject) (providerserver.DeleteProviderResponseObject, error) { - log := logging.FromContext(ctx) - log.Debug("DeleteProvider request received", "provider_id", request.ProviderId) - - err := h.providerService.DeleteProvider(ctx, request.ProviderId) - if err != nil { - logServiceError(ctx, "DeleteProvider failed", err, "provider_id", request.ProviderId) - if svcErr, ok := err.(*service.ServiceError); ok && svcErr.Code == service.ErrCodeNotFound { - return providerserver.DeleteProvider404ApplicationProblemPlusJSONResponse(newError("not-found", "Provider not found", svcErr.Message, 404)), nil - } - return providerserver.DeleteProvider400ApplicationProblemPlusJSONResponse(newError("delete-error", "Failed to delete provider", err.Error(), 400)), nil - } - - log.Info("Provider deleted", "provider_id", request.ProviderId) - return providerserver.DeleteProvider204Response{}, nil -} - -// logServiceError logs at Warn level for client errors (4xx) and Error level -// for internal failures (5xx), so log severity matches HTTP response semantics. -func logServiceError(ctx context.Context, msg string, err error, attrs ...any) { - log := logging.FromContext(ctx) - args := append([]any{"error", err}, attrs...) - var svcErr *service.ServiceError - if service.IsClientError(err, &svcErr) { - log.Warn(msg, args...) - } else { - log.Error(msg, args...) - } -} - -func newError(errType, title, detail string, status int) providerserver.Error { - return providerserver.Error{ - Type: errType, - Title: title, - Detail: &detail, - Status: &status, - } -} diff --git a/internal/sp/handlers/provider/handler_test.go b/internal/sp/handlers/provider/handler_test.go deleted file mode 100644 index ffcf503..0000000 --- a/internal/sp/handlers/provider/handler_test.go +++ /dev/null @@ -1,284 +0,0 @@ -package provider_test - -import ( - "context" - - providerserver "github.com/dcm-project/control-plane/internal/sp/api/provider" - providerhandler "github.com/dcm-project/control-plane/internal/sp/handlers/provider" - providersvc "github.com/dcm-project/control-plane/internal/sp/service/provider" - "github.com/dcm-project/control-plane/internal/sp/store" - "github.com/dcm-project/control-plane/internal/sp/store/model" - "github.com/google/uuid" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "gorm.io/driver/sqlite" - "gorm.io/gorm" - "gorm.io/gorm/logger" -) - -var _ = Describe("Handler", func() { - var ( - db *gorm.DB - handler *providerhandler.Handler - ctx context.Context - ) - - BeforeEach(func() { - var err error - db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ - Logger: logger.Default.LogMode(logger.Silent), - }) - Expect(err).NotTo(HaveOccurred()) - Expect(db.AutoMigrate(&model.Provider{})).To(Succeed()) - - dataStore := store.NewStore(db) - providerService := providersvc.NewProviderService(dataStore) - handler = providerhandler.NewHandler(providerService) - ctx = context.Background() - }) - - AfterEach(func() { - sqlDB, _ := db.DB() - _ = sqlDB.Close() - }) - - Describe("CreateProvider", func() { - It("creates and returns 201", func() { - req := providerserver.CreateProviderRequestObject{ - Body: &providerserver.Provider{ - Name: "test-provider", - Endpoint: "https://example.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }, - } - - resp, err := handler.CreateProvider(ctx, req) - - Expect(err).NotTo(HaveOccurred()) - _, ok := resp.(providerserver.CreateProvider201JSONResponse) - Expect(ok).To(BeTrue()) - }) - - It("returns 200 for idempotent re-registration", func() { - req := providerserver.CreateProviderRequestObject{ - Body: &providerserver.Provider{ - Name: "idempotent-provider", - Endpoint: "https://example.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }, - } - - // First call creates - resp1, err := handler.CreateProvider(ctx, req) - Expect(err).NotTo(HaveOccurred()) - _, ok := resp1.(providerserver.CreateProvider201JSONResponse) - Expect(ok).To(BeTrue()) - - // Second call updates (same name, no ID) - resp2, err := handler.CreateProvider(ctx, req) - Expect(err).NotTo(HaveOccurred()) - _, ok = resp2.(providerserver.CreateProvider200JSONResponse) - Expect(ok).To(BeTrue()) - }) - - It("returns 409 for name conflict with different ID", func() { - // Create first provider - req1 := providerserver.CreateProviderRequestObject{ - Body: &providerserver.Provider{ - Name: "conflict-name", - Endpoint: "https://example.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }, - } - _, err := handler.CreateProvider(ctx, req1) - Expect(err).NotTo(HaveOccurred()) - - // Try to create with same name but different ID - differentID := uuid.New().String() - req2 := providerserver.CreateProviderRequestObject{ - Params: providerserver.CreateProviderParams{Id: &differentID}, - Body: &providerserver.Provider{ - Name: "conflict-name", - Endpoint: "https://other.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }, - } - - resp, err := handler.CreateProvider(ctx, req2) - - Expect(err).NotTo(HaveOccurred()) - _, ok := resp.(providerserver.CreateProvider409ApplicationProblemPlusJSONResponse) - Expect(ok).To(BeTrue()) - }) - }) - - Describe("ListProviders", func() { - It("returns empty list initially", func() { - req := providerserver.ListProvidersRequestObject{} - - resp, err := handler.ListProviders(ctx, req) - - Expect(err).NotTo(HaveOccurred()) - jsonResp, ok := resp.(providerserver.ListProviders200JSONResponse) - Expect(ok).To(BeTrue()) - Expect(*jsonResp.Providers).To(BeEmpty()) - }) - - It("returns providers", func() { - // Create providers first - for _, name := range []string{"provider-1", "provider-2"} { - createReq := providerserver.CreateProviderRequestObject{ - Body: &providerserver.Provider{ - Name: name, - Endpoint: "https://example.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }, - } - _, err := handler.CreateProvider(ctx, createReq) - Expect(err).NotTo(HaveOccurred()) - } - - resp, err := handler.ListProviders(ctx, providerserver.ListProvidersRequestObject{}) - - Expect(err).NotTo(HaveOccurred()) - jsonResp, ok := resp.(providerserver.ListProviders200JSONResponse) - Expect(ok).To(BeTrue()) - Expect(*jsonResp.Providers).To(HaveLen(2)) - }) - }) - - Describe("GetProvider", func() { - It("returns provider", func() { - // Create a provider first - createReq := providerserver.CreateProviderRequestObject{ - Body: &providerserver.Provider{ - Name: "get-me", - Endpoint: "https://example.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }, - } - createResp, _ := handler.CreateProvider(ctx, createReq) - created := createResp.(providerserver.CreateProvider201JSONResponse) - - req := providerserver.GetProviderRequestObject{ - ProviderId: *created.Id, - } - - resp, err := handler.GetProvider(ctx, req) - - Expect(err).NotTo(HaveOccurred()) - jsonResp, ok := resp.(providerserver.GetProvider200JSONResponse) - Expect(ok).To(BeTrue()) - Expect(jsonResp.Name).To(Equal("get-me")) - }) - - It("returns 404 for non-existent provider", func() { - req := providerserver.GetProviderRequestObject{ - ProviderId: uuid.New().String(), - } - - resp, err := handler.GetProvider(ctx, req) - - Expect(err).NotTo(HaveOccurred()) - _, ok := resp.(providerserver.GetProvider404ApplicationProblemPlusJSONResponse) - Expect(ok).To(BeTrue()) - }) - }) - - Describe("ApplyProvider", func() { - It("updates existing provider", func() { - // Create a provider first - createReq := providerserver.CreateProviderRequestObject{ - Body: &providerserver.Provider{ - Name: "to-update", - Endpoint: "https://example.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }, - } - createResp, _ := handler.CreateProvider(ctx, createReq) - created := createResp.(providerserver.CreateProvider201JSONResponse) - - // Update it - updateReq := providerserver.ApplyProviderRequestObject{ - ProviderId: *created.Id, - Body: &providerserver.Provider{ - Id: created.Id, - Name: "to-update", - Endpoint: "https://updated.example.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }, - } - - resp, err := handler.ApplyProvider(ctx, updateReq) - - Expect(err).NotTo(HaveOccurred()) - jsonResp, ok := resp.(providerserver.ApplyProvider200JSONResponse) - Expect(ok).To(BeTrue()) - Expect(jsonResp.Endpoint).To(Equal("https://updated.example.com")) - }) - - It("returns 404 for non-existent provider", func() { - req := providerserver.ApplyProviderRequestObject{ - ProviderId: uuid.New().String(), - Body: &providerserver.Provider{ - Name: "test", - Endpoint: "https://example.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }, - } - - resp, err := handler.ApplyProvider(ctx, req) - - Expect(err).NotTo(HaveOccurred()) - _, ok := resp.(providerserver.ApplyProvider404ApplicationProblemPlusJSONResponse) - Expect(ok).To(BeTrue()) - }) - }) - - Describe("DeleteProvider", func() { - It("deletes provider and returns 204", func() { - // Create a provider first - createReq := providerserver.CreateProviderRequestObject{ - Body: &providerserver.Provider{ - Name: "to-delete", - Endpoint: "https://example.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }, - } - createResp, _ := handler.CreateProvider(ctx, createReq) - created := createResp.(providerserver.CreateProvider201JSONResponse) - - req := providerserver.DeleteProviderRequestObject{ - ProviderId: *created.Id, - } - - resp, err := handler.DeleteProvider(ctx, req) - - Expect(err).NotTo(HaveOccurred()) - _, ok := resp.(providerserver.DeleteProvider204Response) - Expect(ok).To(BeTrue()) - }) - - It("returns 404 for non-existent provider", func() { - req := providerserver.DeleteProviderRequestObject{ - ProviderId: uuid.New().String(), - } - - resp, err := handler.DeleteProvider(ctx, req) - - Expect(err).NotTo(HaveOccurred()) - _, ok := resp.(providerserver.DeleteProvider404ApplicationProblemPlusJSONResponse) - Expect(ok).To(BeTrue()) - }) - }) -}) diff --git a/internal/sp/handlers/resource_manager/convert.go b/internal/sp/handlers/resource_manager/convert.go index 29b6fdc..9411982 100644 --- a/internal/sp/handlers/resource_manager/convert.go +++ b/internal/sp/handlers/resource_manager/convert.go @@ -8,22 +8,21 @@ import ( // convertServerToAPI converts a server ServiceTypeInstance to an API ServiceTypeInstance. func convertServerToAPI(src *server.ServiceTypeInstance) *resource_manager.ServiceTypeInstance { return &resource_manager.ServiceTypeInstance{ - Id: src.Id, - ProviderName: src.ProviderName, - Spec: src.Spec, + Id: src.Id, + Spec: src.Spec, } } // convertAPIToServer converts an API ServiceTypeInstance to a server ServiceTypeInstance. func convertAPIToServer(src *resource_manager.ServiceTypeInstance) server.ServiceTypeInstance { result := server.ServiceTypeInstance{ - Id: src.Id, - Path: src.Path, - ProviderName: src.ProviderName, - Status: src.Status, - Spec: src.Spec, - CreateTime: src.CreateTime, - UpdateTime: src.UpdateTime, + Id: src.Id, + Path: src.Path, + AgentName: src.AgentName, + Status: src.Status, + Spec: src.Spec, + CreateTime: src.CreateTime, + UpdateTime: src.UpdateTime, } if src.DeletionStatus != nil { diff --git a/internal/sp/handlers/resource_manager/errors.go b/internal/sp/handlers/resource_manager/errors.go index 77a678a..e23f159 100644 --- a/internal/sp/handlers/resource_manager/errors.go +++ b/internal/sp/handlers/resource_manager/errors.go @@ -32,6 +32,13 @@ func newError(errType, title, detail string, status int) server.Error { } } +// internalErrorDetail is the fixed 5xx response detail used across this +// file. The real error (which may contain DB/NATS/internal detail) is +// always logged server-side by logServiceError before these functions are +// called, so the client-facing body never echoes it back (F37/P: 5xx bodies +// must not leak internal error strings). +const internalErrorDetail = "an internal error occurred" + // handleListInstancesError converts a service error to a ListInstances response. func handleListInstancesError(err error) server.ListInstancesResponseObject { var svcErr *service.ServiceError @@ -39,7 +46,7 @@ func handleListInstancesError(err error) server.ListInstancesResponseObject { return server.ListInstances400ApplicationProblemPlusJSONResponse(newError("validation-error", "Invalid request", svcErr.Message, 400)) } return server.ListInstancesdefaultApplicationProblemPlusJSONResponse{ - Body: newError("list-error", "Failed to list instances", err.Error(), 500), + Body: newError("list-error", "Failed to list instances", internalErrorDetail, 500), StatusCode: 500, } } @@ -55,17 +62,19 @@ func handleCreateInstanceError(err error) server.CreateInstanceResponseObject { return server.CreateInstance404ApplicationProblemPlusJSONResponse(newError("not-found", "Resource not found", svcErr.Message, 404)) case service.ErrCodeConflict: return server.CreateInstance409ApplicationProblemPlusJSONResponse(newError("conflict", "Resource conflict", svcErr.Message, 409)) - case service.ErrCodeProviderError: - return server.CreateInstance422ApplicationProblemPlusJSONResponse(newError("provider-error", "Provider error", svcErr.Message, 422)) + case service.ErrCodeProvisioningError: + return server.CreateInstance422ApplicationProblemPlusJSONResponse(newError("provisioning-error", "Provisioning error", svcErr.Message, 422)) case service.ErrCodeInternal: return server.CreateInstancedefaultApplicationProblemPlusJSONResponse{ - Body: newError("internal-error", "Internal error", svcErr.Message, 500), + Body: newError("internal-error", "Internal error", internalErrorDetail, 500), StatusCode: 500, } + case service.ErrCodeUnavailable: + return server.CreateInstance503ApplicationProblemPlusJSONResponse(newError("unavailable", "Service unavailable", "service temporarily unavailable", 503)) } } return server.CreateInstancedefaultApplicationProblemPlusJSONResponse{ - Body: newError("create-error", "Failed to create instance", err.Error(), 500), + Body: newError("create-error", "Failed to create instance", internalErrorDetail, 500), StatusCode: 500, } } @@ -82,7 +91,7 @@ func handleGetInstanceError(err error) server.GetInstanceResponseObject { } } return server.GetInstancedefaultApplicationProblemPlusJSONResponse{ - Body: newError("get-error", "Failed to get instance", err.Error(), 500), + Body: newError("get-error", "Failed to get instance", internalErrorDetail, 500), StatusCode: 500, } } @@ -96,10 +105,14 @@ func handleDeleteInstanceError(err error) server.DeleteInstanceResponseObject { return server.DeleteInstance400ApplicationProblemPlusJSONResponse(newError("validation-error", "Invalid request", svcErr.Message, 400)) case service.ErrCodeNotFound: return server.DeleteInstance404ApplicationProblemPlusJSONResponse(newError("not-found", "Instance not found", svcErr.Message, 404)) + case service.ErrCodeProvisioningError: + // A transient, client-actionable failure to publish the delete, + // not an internal server bug - map to 422 like CreateInstance. + return server.DeleteInstance422ApplicationProblemPlusJSONResponse(newError("provisioning-error", "Provisioning error", svcErr.Message, 422)) } } return server.DeleteInstancedefaultApplicationProblemPlusJSONResponse{ - Body: newError("delete-error", "Failed to delete instance", err.Error(), 500), + Body: newError("delete-error", "Failed to delete instance", internalErrorDetail, 500), StatusCode: 500, } } diff --git a/internal/sp/handlers/resource_manager/errors_test.go b/internal/sp/handlers/resource_manager/errors_test.go new file mode 100644 index 0000000..3150d13 --- /dev/null +++ b/internal/sp/handlers/resource_manager/errors_test.go @@ -0,0 +1,29 @@ +package resource_manager + +import ( + server "github.com/dcm-project/control-plane/internal/sp/api/resource_manager" + "github.com/dcm-project/control-plane/internal/sp/service" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("handleDeleteInstanceError", func() { + It("maps a ProvisioningError to 422 instead of the generic 500 default (R2 S7: finding #1)", func() { + // DeleteInstance's non-deferred path returns this when publishing + // the delete to the agent fails: a transient, client-actionable + // failure to carry out the delete, not an internal server bug. + resp := handleDeleteInstanceError(service.NewProvisioningError("failed to publish delete for instance x: nats unavailable")) + + typedResp, ok := resp.(server.DeleteInstance422ApplicationProblemPlusJSONResponse) + Expect(ok).To(BeTrue()) + Expect(typedResp.Type).To(Equal("provisioning-error")) + }) + + It("still maps unrecognized errors to the generic 500 default", func() { + resp := handleDeleteInstanceError(service.NewInternalError("boom")) + + defResp, ok := resp.(server.DeleteInstancedefaultApplicationProblemPlusJSONResponse) + Expect(ok).To(BeTrue()) + Expect(defResp.StatusCode).To(Equal(500)) + }) +}) diff --git a/internal/sp/handlers/resource_manager/handler.go b/internal/sp/handlers/resource_manager/handler.go index 116aedf..67959a6 100644 --- a/internal/sp/handlers/resource_manager/handler.go +++ b/internal/sp/handlers/resource_manager/handler.go @@ -27,15 +27,14 @@ func (h *Handler) ListInstances(ctx context.Context, request server.ListInstance log := logging.FromContext(ctx) showDeleted := request.Params.ShowDeleted != nil && *request.Params.ShowDeleted log.Debug("ListInstances request received", - "provider", request.Params.Provider, "page_size", request.Params.MaxPageSize, "show_deleted", showDeleted, ) result, err := h.instanceService.ListInstances( ctx, - request.Params.Provider, request.Params.ServiceType, + request.Params.AgentName, showDeleted, request.Params.MaxPageSize, request.Params.PageToken, @@ -58,14 +57,11 @@ func (h *Handler) ListInstances(ctx context.Context, request server.ListInstance // CreateInstance creates a new service type instance. func (h *Handler) CreateInstance(ctx context.Context, request server.CreateInstanceRequestObject) (server.CreateInstanceResponseObject, error) { log := logging.FromContext(ctx) - log.Debug("CreateInstance request received", - "client_id", request.Params.Id, - "provider_name", request.Body.ProviderName, - ) + log.Debug("CreateInstance request received", "client_id", request.Params.Id) instance := convertServerToAPI(request.Body) - result, err := h.instanceService.CreateInstance(ctx, instance, request.Params.Id) + result, err := h.instanceService.CreateInstance(ctx, instance, request.Params.Id, "") if err != nil { logServiceError(ctx, "CreateInstance failed", err) return handleCreateInstanceError(err), nil diff --git a/internal/sp/handlers/resource_manager/handler_test.go b/internal/sp/handlers/resource_manager/handler_test.go index ee4f9ca..3349ca3 100644 --- a/internal/sp/handlers/resource_manager/handler_test.go +++ b/internal/sp/handlers/resource_manager/handler_test.go @@ -2,17 +2,13 @@ package resource_manager_test import ( "context" - "encoding/json" - "net/http" - "net/http/httptest" - "time" + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" server "github.com/dcm-project/control-plane/internal/sp/api/resource_manager" rmhandlers "github.com/dcm-project/control-plane/internal/sp/handlers/resource_manager" rmsvc "github.com/dcm-project/control-plane/internal/sp/service/resource_manager" "github.com/dcm-project/control-plane/internal/sp/store" "github.com/dcm-project/control-plane/internal/sp/store/model" - "github.com/go-resty/resty/v2" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,11 +19,9 @@ import ( var _ = Describe("Resource Manager Handler", func() { var ( - db *gorm.DB - handler *rmhandlers.Handler - ctx context.Context - mockProvider *httptest.Server - providerCalled bool + db *gorm.DB + handler *rmhandlers.Handler + ctx context.Context ) BeforeEach(func() { @@ -36,123 +30,41 @@ var _ = Describe("Resource Manager Handler", func() { Logger: logger.Default.LogMode(logger.Silent), }) Expect(err).NotTo(HaveOccurred()) - Expect(db.AutoMigrate(&model.Provider{}, &model.ServiceTypeInstance{})).To(Succeed()) - - // Create a mock provider server - providerCalled = false - mockProvider = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - providerCalled = true - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _ = json.NewEncoder(w).Encode(map[string]string{ - "id": uuid.New().String(), - "status": "PROVISIONING", - }) - })) - - // Create a provider in the database - provider := model.Provider{ - ID: uuid.New().String(), - Name: "test-provider", - ServiceType: "vm", - Endpoint: mockProvider.URL, - } - Expect(db.Create(&provider).Error).NotTo(HaveOccurred()) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) dataStore := store.NewStore(db) - instanceService := rmsvc.NewInstanceService(dataStore, resty.New(). - SetTimeout(5*time.Second). - SetRetryCount(0)) + instanceService := rmsvc.NewInstanceService(dataStore, nil, nil) handler = rmhandlers.NewHandler(instanceService) ctx = context.Background() }) AfterEach(func() { - mockProvider.Close() sqlDB, _ := db.DB() _ = sqlDB.Close() }) Describe("CreateInstance", func() { - It("creates and returns 201", func() { - req := server.CreateInstanceRequestObject{ - Body: &server.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "memory": "4GB", "service_type": "vm"}, - }, - } - - resp, err := handler.CreateInstance(ctx, req) - - Expect(err).NotTo(HaveOccurred()) - jsonResp, ok := resp.(server.CreateInstance201JSONResponse) - Expect(ok).To(BeTrue()) - Expect(jsonResp.ProviderName).To(Equal("test-provider")) - Expect(jsonResp.Id).NotTo(BeNil()) - Expect(providerCalled).To(BeTrue()) - }) - - It("creates with specified ID", func() { - specifiedID := uuid.New().String() - req := server.CreateInstanceRequestObject{ - Params: server.CreateInstanceParams{Id: &specifiedID}, - Body: &server.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, - }, - } - - resp, err := handler.CreateInstance(ctx, req) - - Expect(err).NotTo(HaveOccurred()) - jsonResp, ok := resp.(server.CreateInstance201JSONResponse) - Expect(ok).To(BeTrue()) - Expect(*jsonResp.Id).To(Equal(specifiedID)) - }) - - It("returns 409 for duplicate ID", func() { - specifiedID := uuid.New().String() - req := server.CreateInstanceRequestObject{ - Params: server.CreateInstanceParams{Id: &specifiedID}, - Body: &server.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, - }, - } - - // First creation should succeed - resp1, err := handler.CreateInstance(ctx, req) - Expect(err).NotTo(HaveOccurred()) - _, ok := resp1.(server.CreateInstance201JSONResponse) - Expect(ok).To(BeTrue()) - - // Second creation with same ID should fail - resp2, err := handler.CreateInstance(ctx, req) - Expect(err).NotTo(HaveOccurred()) - _, ok = resp2.(server.CreateInstance409ApplicationProblemPlusJSONResponse) - Expect(ok).To(BeTrue()) - }) - - It("returns 404 for non-existent provider", func() { + // This endpoint never receives an agent_name from the caller - it's + // resolved upstream by placement/SPRM - so it always calls + // CreateInstance with agentName="", which must be rejected. + It("returns 400 because this endpoint never supplies an agent name", func() { req := server.CreateInstanceRequestObject{ Body: &server.ServiceTypeInstance{ - ProviderName: "non-existent-provider", - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, + Spec: map[string]interface{}{"cpu": 2, "memory": "4GB", "service_type": "vm"}, }, } resp, err := handler.CreateInstance(ctx, req) Expect(err).NotTo(HaveOccurred()) - _, ok := resp.(server.CreateInstance404ApplicationProblemPlusJSONResponse) + _, ok := resp.(server.CreateInstance400ApplicationProblemPlusJSONResponse) Expect(ok).To(BeTrue()) }) It("returns 400 when spec is missing service_type", func() { req := server.CreateInstanceRequestObject{ Body: &server.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2}, + Spec: map[string]interface{}{"cpu": 2}, }, } @@ -161,14 +73,12 @@ var _ = Describe("Resource Manager Handler", func() { Expect(err).NotTo(HaveOccurred()) _, ok := resp.(server.CreateInstance400ApplicationProblemPlusJSONResponse) Expect(ok).To(BeTrue()) - Expect(providerCalled).To(BeFalse()) }) It("returns 400 when spec.service_type is not a string", func() { req := server.CreateInstanceRequestObject{ Body: &server.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": 42}, + Spec: map[string]interface{}{"cpu": 2, "service_type": 42}, }, } @@ -177,14 +87,12 @@ var _ = Describe("Resource Manager Handler", func() { Expect(err).NotTo(HaveOccurred()) _, ok := resp.(server.CreateInstance400ApplicationProblemPlusJSONResponse) Expect(ok).To(BeTrue()) - Expect(providerCalled).To(BeFalse()) }) It("returns 400 when spec.service_type is an empty string", func() { req := server.CreateInstanceRequestObject{ Body: &server.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": ""}, + Spec: map[string]interface{}{"cpu": 2, "service_type": ""}, }, } @@ -193,24 +101,21 @@ var _ = Describe("Resource Manager Handler", func() { Expect(err).NotTo(HaveOccurred()) _, ok := resp.(server.CreateInstance400ApplicationProblemPlusJSONResponse) Expect(ok).To(BeTrue()) - Expect(providerCalled).To(BeFalse()) }) }) Describe("GetInstance", func() { It("returns instance", func() { - // Create an instance first - createReq := server.CreateInstanceRequestObject{ - Body: &server.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, - }, - } - createResp, _ := handler.CreateInstance(ctx, createReq) - created := createResp.(server.CreateInstance201JSONResponse) + instanceID := uuid.New().String() + db.Create(&model.ServiceTypeInstance{ + ID: instanceID, + ServiceType: "vm", + Status: "pending", + Spec: map[string]any{"cpu": 2, "service_type": "vm"}, + }) req := server.GetInstanceRequestObject{ - InstanceId: *created.Id, + InstanceId: instanceID, } resp, err := handler.GetInstance(ctx, req) @@ -218,7 +123,7 @@ var _ = Describe("Resource Manager Handler", func() { Expect(err).NotTo(HaveOccurred()) jsonResp, ok := resp.(server.GetInstance200JSONResponse) Expect(ok).To(BeTrue()) - Expect(jsonResp.ProviderName).To(Equal("test-provider")) + Expect(*jsonResp.Id).To(Equal(instanceID)) }) It("returns 404 for non-existent instance", func() { @@ -247,16 +152,13 @@ var _ = Describe("Resource Manager Handler", func() { }) It("returns instances", func() { - // Create instances first for i := 0; i < 3; i++ { - createReq := server.CreateInstanceRequestObject{ - Body: &server.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": i + 1, "service_type": "vm"}, - }, - } - _, err := handler.CreateInstance(ctx, createReq) - Expect(err).NotTo(HaveOccurred()) + db.Create(&model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "pending", + Spec: map[string]any{"cpu": 1, "service_type": "vm"}, + }) } resp, err := handler.ListInstances(ctx, server.ListInstancesRequestObject{}) @@ -267,67 +169,53 @@ var _ = Describe("Resource Manager Handler", func() { Expect(*jsonResp.Instances).To(HaveLen(3)) }) - It("filters by service_type", func() { - containerProvider := model.Provider{ - ID: uuid.New().String(), - Name: "container-provider", - ServiceType: "container", - Endpoint: mockProvider.URL, - } - Expect(db.Create(&containerProvider).Error).NotTo(HaveOccurred()) - - // Create vm instances with service_type in spec - for i := 0; i < 2; i++ { - createReq := server.CreateInstanceRequestObject{ - Body: &server.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": i + 1, "service_type": "vm"}, - }, - } - _, err := handler.CreateInstance(ctx, createReq) - Expect(err).NotTo(HaveOccurred()) - } - - // Create container instance with service_type in spec - createReq := server.CreateInstanceRequestObject{ - Body: &server.ServiceTypeInstance{ - ProviderName: "container-provider", - Spec: map[string]interface{}{"image": "nginx", "service_type": "container"}, - }, - } - _, err := handler.CreateInstance(ctx, createReq) - Expect(err).NotTo(HaveOccurred()) + It("filters by service type and agent name independently, without swapping them", func() { + agentA, agentB := "agent-a", "agent-b" + vmID, dbID := uuid.New().String(), uuid.New().String() + db.Create(&model.ServiceTypeInstance{ + ID: vmID, + ServiceType: "vm", + AgentName: &agentA, + Status: "pending", + Spec: map[string]any{"cpu": 1, "service_type": "vm"}, + }) + db.Create(&model.ServiceTypeInstance{ + ID: dbID, + ServiceType: "db", + AgentName: &agentB, + Status: "pending", + Spec: map[string]any{"cpu": 1, "service_type": "db"}, + }) - vmType := "vm" + serviceType := "vm" resp, err := handler.ListInstances(ctx, server.ListInstancesRequestObject{ - Params: server.ListInstancesParams{ServiceType: &vmType}, + Params: server.ListInstancesParams{ServiceType: &serviceType}, }) Expect(err).NotTo(HaveOccurred()) jsonResp, ok := resp.(server.ListInstances200JSONResponse) Expect(ok).To(BeTrue()) - Expect(*jsonResp.Instances).To(HaveLen(2)) + Expect(*jsonResp.Instances).To(HaveLen(1)) + Expect(*(*jsonResp.Instances)[0].Id).To(Equal(vmID)) - containerType := "container" resp, err = handler.ListInstances(ctx, server.ListInstancesRequestObject{ - Params: server.ListInstancesParams{ServiceType: &containerType}, + Params: server.ListInstancesParams{AgentName: &agentB}, }) Expect(err).NotTo(HaveOccurred()) jsonResp, ok = resp.(server.ListInstances200JSONResponse) Expect(ok).To(BeTrue()) Expect(*jsonResp.Instances).To(HaveLen(1)) + Expect(*(*jsonResp.Instances)[0].Id).To(Equal(dbID)) + Expect(*(*jsonResp.Instances)[0].AgentName).To(Equal(agentB)) }) It("respects max page size and returns next page token", func() { - // Create 5 instances for i := 0; i < 5; i++ { - createReq := server.CreateInstanceRequestObject{ - Body: &server.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": i + 1, "service_type": "vm"}, - }, - } - _, err := handler.CreateInstance(ctx, createReq) - Expect(err).NotTo(HaveOccurred()) + db.Create(&model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "pending", + Spec: map[string]any{"cpu": 1, "service_type": "vm"}, + }) } // First page: request 2 items @@ -381,18 +269,16 @@ var _ = Describe("Resource Manager Handler", func() { Describe("DeleteInstance", func() { It("deletes instance and returns 204", func() { - // Create an instance first - createReq := server.CreateInstanceRequestObject{ - Body: &server.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, - }, - } - createResp, _ := handler.CreateInstance(ctx, createReq) - created := createResp.(server.CreateInstance201JSONResponse) + instanceID := uuid.New().String() + db.Create(&model.ServiceTypeInstance{ + ID: instanceID, + ServiceType: "vm", + Status: "pending", + Spec: map[string]any{"cpu": 2, "service_type": "vm"}, + }) req := server.DeleteInstanceRequestObject{ - InstanceId: *created.Id, + InstanceId: instanceID, } resp, err := handler.DeleteInstance(ctx, req) @@ -402,7 +288,7 @@ var _ = Describe("Resource Manager Handler", func() { Expect(ok).To(BeTrue()) // Verify it's deleted - getResp, _ := handler.GetInstance(ctx, server.GetInstanceRequestObject{InstanceId: *created.Id}) + getResp, _ := handler.GetInstance(ctx, server.GetInstanceRequestObject{InstanceId: instanceID}) _, ok = getResp.(server.GetInstance404ApplicationProblemPlusJSONResponse) Expect(ok).To(BeTrue()) }) diff --git a/internal/sp/healthcheck/monitor.go b/internal/sp/healthcheck/monitor.go deleted file mode 100644 index 7082395..0000000 --- a/internal/sp/healthcheck/monitor.go +++ /dev/null @@ -1,216 +0,0 @@ -// Package healthcheck performs periodic health checks on registered service providers. -package healthcheck - -import ( - "context" - "encoding/json" - "log/slog" - "math" - "net/http" - "strings" - "sync" - "time" - - "github.com/dcm-project/control-plane/internal/sp/config" - "github.com/dcm-project/control-plane/internal/sp/store/model" - providerstore "github.com/dcm-project/control-plane/internal/sp/store/provider" - rmstore "github.com/dcm-project/control-plane/internal/sp/store/resource_manager" -) - -type healthCheckResult int - -const ( - healthCheckHealthy healthCheckResult = iota - healthCheckUnhealthy - healthCheckFailed -) - -type healthResponse struct { - Status string `json:"status"` -} - -// Monitor performs periodic health checks on registered service providers -type Monitor struct { - store providerstore.Provider - instanceStore rmstore.ServiceTypeInstance - httpClient *http.Client - interval time.Duration - stopCh chan struct{} - wg sync.WaitGroup - maxConsecutiveFailures int - baseBackoffInterval time.Duration - maxBackoffInterval time.Duration -} - -// NewMonitor creates a new health check monitor -func NewMonitor(providerStore providerstore.Provider, instanceStore rmstore.ServiceTypeInstance, config *config.HealthCheckConfig) *Monitor { - return &Monitor{ - store: providerStore, - instanceStore: instanceStore, - httpClient: &http.Client{ - Timeout: config.Timeout, - }, - interval: config.Interval, - stopCh: make(chan struct{}), - maxConsecutiveFailures: config.MaxConsecutiveFailures, - baseBackoffInterval: config.BaseBackoffInterval, - maxBackoffInterval: config.MaxBackoffInterval, - } -} - -// Start begins the health check monitoring loop -func (m *Monitor) Start(ctx context.Context) { - m.wg.Add(1) - go m.run(ctx) -} - -// Stop gracefully stops the health check monitor -func (m *Monitor) Stop() { - close(m.stopCh) - m.wg.Wait() -} - -func (m *Monitor) run(ctx context.Context) { - defer m.wg.Done() - - ticker := time.NewTicker(m.interval) - defer ticker.Stop() - - // Run immediately on start - m.CheckProviders(ctx) - - for { - select { - case <-ctx.Done(): - return - case <-m.stopCh: - return - case <-ticker.C: - m.CheckProviders(ctx) - } - } -} - -// CheckProviders checks all providers that are due for a health check -func (m *Monitor) CheckProviders(ctx context.Context) { - now := time.Now() - providers, err := m.store.ListProvidersForHealthCheck(ctx, now) - if err != nil { - slog.Error("Error listing providers for health check", "error", err) - return - } - - for _, provider := range providers { - select { - case <-ctx.Done(): - return - default: - m.checkProvider(ctx, provider) - } - } -} - -func (m *Monitor) checkProvider(ctx context.Context, provider model.Provider) { - now := time.Now() - var newStatus model.HealthStatus - var consecutiveFailures int - - result := m.performHealthCheck(ctx, provider) - switch result { - case healthCheckHealthy: - newStatus = model.HealthStatusReady - consecutiveFailures = 0 - case healthCheckUnhealthy: - newStatus = model.HealthStatusUnhealthy - consecutiveFailures = 0 - case healthCheckFailed: - consecutiveFailures = provider.ConsecutiveFailures + 1 - newStatus = provider.HealthStatus - if consecutiveFailures >= m.maxConsecutiveFailures { - newStatus = model.HealthStatusUnavailable - } - } - - nextCheck := m.CalculateNextCheckTime(now, newStatus, consecutiveFailures) - if err := m.store.UpdateHealthStatus(ctx, provider.ID, newStatus, consecutiveFailures, nextCheck); err != nil { - slog.Error("Error updating health status", "provider", provider.Name, "error", err) - return - } - - if provider.HealthStatus != newStatus { - slog.Info("Provider health status changed", - "provider", provider.Name, - "old_status", provider.HealthStatus, - "new_status", newStatus, - ) - - switch newStatus { - case model.HealthStatusUnhealthy, model.HealthStatusUnavailable: - if err := m.instanceStore.MarkProviderDeletionsPendingProvider(ctx, provider.Name); err != nil { - slog.Error("Failed to park deletions for unhealthy provider", "provider", provider.Name, "error", err) - } - case model.HealthStatusReady: - if err := m.instanceStore.ReactivateProviderDeletions(ctx, provider.Name); err != nil { - slog.Error("Failed to reactivate deletions for recovered provider", "provider", provider.Name, "error", err) - } - } - } -} - -func (m *Monitor) performHealthCheck(ctx context.Context, provider model.Provider) healthCheckResult { - healthURL := strings.TrimRight(provider.Endpoint, "/") + "/health" - req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthURL, nil) - if err != nil { - slog.Error("Error creating health check request", "provider", provider.Name, "error", err) - return healthCheckFailed - } - - resp, err := m.httpClient.Do(req) - if err != nil { - slog.Debug("Health check failed", "provider", provider.Name, "error", err) - return healthCheckFailed - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - slog.Debug("Health check failed", "provider", provider.Name, "status_code", resp.StatusCode) - return healthCheckFailed - } - - var hr healthResponse - if err := json.NewDecoder(resp.Body).Decode(&hr); err != nil { - slog.Error("Error parsing health check response", "provider", provider.Name, "error", err) - return healthCheckFailed - } - - switch hr.Status { - case "healthy": - return healthCheckHealthy - case "unhealthy": - slog.Debug("Provider reports unhealthy backing provider", "provider", provider.Name) - return healthCheckUnhealthy - default: - slog.Debug("Unknown health status in response", "provider", provider.Name, "status", hr.Status) - return healthCheckFailed - } -} - -// CalculateNextCheckTime determines when the next health check should occur -// For Ready and Unhealthy providers: standard interval (provider is reachable) -// Exponential backoff for Unavailable providers -// Formula: min(MaxBackoff, BaseInterval * 2^(failures - MaxConsecutiveFailures)) -func (m *Monitor) CalculateNextCheckTime(now time.Time, status model.HealthStatus, consecutiveFailures int) time.Time { - if status != model.HealthStatusUnavailable { - return now.Add(m.interval) - } - - exponent := max(consecutiveFailures-m.maxConsecutiveFailures, 0) - - const maxExponent = 10 - exponent = min(exponent, maxExponent) - - backoffMultiplier := math.Pow(2, float64(exponent)) - backoffDuration := min(time.Duration(float64(m.baseBackoffInterval)*backoffMultiplier), m.maxBackoffInterval) - - return now.Add(backoffDuration) -} diff --git a/internal/sp/healthcheck/monitor_test.go b/internal/sp/healthcheck/monitor_test.go deleted file mode 100644 index 45a7141..0000000 --- a/internal/sp/healthcheck/monitor_test.go +++ /dev/null @@ -1,601 +0,0 @@ -package healthcheck_test - -import ( - "context" - "fmt" - "net/http" - "net/http/httptest" - "time" - - "github.com/dcm-project/control-plane/internal/sp/config" - "github.com/dcm-project/control-plane/internal/sp/healthcheck" - "github.com/dcm-project/control-plane/internal/sp/store/model" - providerstore "github.com/dcm-project/control-plane/internal/sp/store/provider" - rmstore "github.com/dcm-project/control-plane/internal/sp/store/resource_manager" - "github.com/google/uuid" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -// testHealthCheckConfig returns a default config for testing -func testHealthCheckConfig() *config.HealthCheckConfig { - return &config.HealthCheckConfig{ - Interval: 10 * time.Second, - Timeout: 5 * time.Second, - MaxConsecutiveFailures: 3, - BaseBackoffInterval: 10 * time.Second, - MaxBackoffInterval: 5 * time.Minute, - } -} - -// mockProviderStore implements store.Provider interface for testing -type mockProviderStore struct { - providers model.ProviderList - healthStatusUpdates []healthStatusUpdate -} - -type healthStatusUpdate struct { - ID string - Status model.HealthStatus - ConsecutiveFailures int - NextCheck time.Time -} - -func (m *mockProviderStore) ListProvidersForHealthCheck(_ context.Context, now time.Time) (model.ProviderList, error) { - var result model.ProviderList - for _, p := range m.providers { - if p.NextHealthCheck == nil || !p.NextHealthCheck.After(now) { - result = append(result, p) - } - } - return result, nil -} - -func (m *mockProviderStore) UpdateHealthStatus(_ context.Context, id string, status model.HealthStatus, consecutiveFailures int, nextCheck time.Time) error { - m.healthStatusUpdates = append(m.healthStatusUpdates, healthStatusUpdate{ - ID: id, - Status: status, - ConsecutiveFailures: consecutiveFailures, - NextCheck: nextCheck, - }) - return nil -} - -func (m *mockProviderStore) List(_ context.Context, _ *providerstore.ProviderFilter, _ *providerstore.Pagination) (model.ProviderList, error) { - return m.providers, nil -} - -func (m *mockProviderStore) Count(_ context.Context, _ *providerstore.ProviderFilter) (int64, error) { - return int64(len(m.providers)), nil -} - -func (m *mockProviderStore) ExistsByID(_ context.Context, id string) (bool, error) { - for _, p := range m.providers { - if p.ID == id { - return true, nil - } - } - return false, nil -} - -func (m *mockProviderStore) Create(_ context.Context, provider model.Provider) (*model.Provider, error) { - return &provider, nil -} - -func (m *mockProviderStore) Delete(_ context.Context, _ string) error { - return nil -} - -func (m *mockProviderStore) Update(_ context.Context, provider model.Provider) (*model.Provider, error) { - return &provider, nil -} - -func (m *mockProviderStore) Get(_ context.Context, id string) (*model.Provider, error) { - for _, p := range m.providers { - if p.ID == id { - return &p, nil - } - } - return nil, nil -} - -func (m *mockProviderStore) GetByName(_ context.Context, name string) (*model.Provider, error) { - for _, p := range m.providers { - if p.Name == name { - return &p, nil - } - } - return nil, nil -} - -// mockInstanceStore implements rmstore.ServiceTypeInstance for testing -type mockInstanceStore struct { - markProviderDeletionsCalls []string - reactivateProviderCalls []string - markPendingProviderCalls []string - markPendingProviderResult bool - markPendingProviderErr error -} - -func (m *mockInstanceStore) MarkProviderDeletionsPendingProvider(_ context.Context, providerName string) error { - m.markProviderDeletionsCalls = append(m.markProviderDeletionsCalls, providerName) - return nil -} - -func (m *mockInstanceStore) ReactivateProviderDeletions(_ context.Context, providerName string) error { - m.reactivateProviderCalls = append(m.reactivateProviderCalls, providerName) - return nil -} - -func (m *mockInstanceStore) MarkPendingProviderIfNotReady(_ context.Context, instanceID string) (bool, error) { - m.markPendingProviderCalls = append(m.markPendingProviderCalls, instanceID) - return m.markPendingProviderResult, m.markPendingProviderErr -} - -// Unused interface methods -func (m *mockInstanceStore) List(_ context.Context, _ *rmstore.ServiceTypeInstanceListOptions) (*rmstore.ServiceTypeInstanceListResult, error) { - return nil, nil -} - -func (m *mockInstanceStore) Create(_ context.Context, inst model.ServiceTypeInstance) (*model.ServiceTypeInstance, error) { - return &inst, nil -} - -func (m *mockInstanceStore) Get(_ context.Context, _ string, _ bool) (*model.ServiceTypeInstance, error) { - return nil, nil -} - -func (m *mockInstanceStore) ExistsByID(_ context.Context, _ string) (bool, error) { return false, nil } - -func (m *mockInstanceStore) UpdateStatus(_ context.Context, _ string, _ string, _ string) error { - return nil -} - -func (m *mockInstanceStore) MarkForDeletion(_ context.Context, _ string) error { return nil } - -func (m *mockInstanceStore) ListPendingDeletions(_ context.Context) ([]model.ServiceTypeInstance, error) { - return nil, nil -} - -func (m *mockInstanceStore) IncrementDeletionRetry(_ context.Context, _ string) error { return nil } -func (m *mockInstanceStore) MarkDeletionFailed(_ context.Context, _ string) error { return nil } -func (m *mockInstanceStore) HardDelete(_ context.Context, _ string) error { return nil } -func (m *mockInstanceStore) ResetRetryCount(_ context.Context, _ string) error { return nil } - -func healthyServer() *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/health" { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _, _ = fmt.Fprint(w, `{"status":"healthy"}`) - return - } - w.WriteHeader(http.StatusNotFound) - })) -} - -func unhealthyServer() *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _, _ = fmt.Fprint(w, `{"status":"unhealthy"}`) - })) -} - -func failingServer() *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - })) -} - -var _ = Describe("Monitor", func() { - var ( - cfg *config.HealthCheckConfig - monitor *healthcheck.Monitor - ctx context.Context - ) - - BeforeEach(func() { - cfg = testHealthCheckConfig() - ctx = context.Background() - }) - - Describe("CalculateNextCheckTime", func() { - Context("for a Ready provider", func() { - It("schedules next check at the configured interval", func() { - mockStore := &mockProviderStore{} - monitor = healthcheck.NewMonitor(mockStore, &mockInstanceStore{}, cfg) - now := time.Now() - - nextCheck := monitor.CalculateNextCheckTime(now, model.HealthStatusReady, 0) - - Expect(nextCheck.Sub(now)).To(Equal(cfg.Interval)) - }) - }) - - Context("for an Unhealthy provider", func() { - It("schedules next check at the configured interval", func() { - mockStore := &mockProviderStore{} - monitor = healthcheck.NewMonitor(mockStore, &mockInstanceStore{}, cfg) - now := time.Now() - - nextCheck := monitor.CalculateNextCheckTime(now, model.HealthStatusUnhealthy, 0) - - Expect(nextCheck.Sub(now)).To(Equal(cfg.Interval)) - }) - }) - - Context("for an Unavailable provider with exponential backoff", func() { - var ( - mockStore *mockProviderStore - now time.Time - ) - - BeforeEach(func() { - mockStore = &mockProviderStore{} - monitor = healthcheck.NewMonitor(mockStore, &mockInstanceStore{}, cfg) - now = time.Now() - }) - - It("uses base backoff interval when just became Unavailable (3 failures)", func() { - nextCheck := monitor.CalculateNextCheckTime(now, model.HealthStatusUnavailable, 3) - Expect(nextCheck.Sub(now)).To(Equal(cfg.BaseBackoffInterval)) - }) - - It("doubles backoff for 4 consecutive failures", func() { - nextCheck := monitor.CalculateNextCheckTime(now, model.HealthStatusUnavailable, 4) - Expect(nextCheck.Sub(now)).To(Equal(cfg.BaseBackoffInterval * 2)) - }) - - It("quadruples backoff for 5 consecutive failures", func() { - nextCheck := monitor.CalculateNextCheckTime(now, model.HealthStatusUnavailable, 5) - Expect(nextCheck.Sub(now)).To(Equal(cfg.BaseBackoffInterval * 4)) - }) - - It("caps backoff at max interval for many failures", func() { - nextCheck := monitor.CalculateNextCheckTime(now, model.HealthStatusUnavailable, 100) - Expect(nextCheck.Sub(now)).To(Equal(cfg.MaxBackoffInterval)) - }) - }) - }) - - Describe("CheckProviders", func() { - Context("with a healthy provider", func() { - It("sets status to Ready with zero consecutive failures", func() { - server := healthyServer() - defer server.Close() - - providerID := uuid.New().String() - mockStore := &mockProviderStore{ - providers: model.ProviderList{ - { - ID: providerID, - Name: "test-provider", - Endpoint: server.URL, - HealthStatus: model.HealthStatusReady, - }, - }, - } - - monitor = healthcheck.NewMonitor(mockStore, &mockInstanceStore{}, cfg) - monitor.CheckProviders(ctx) - - Expect(mockStore.healthStatusUpdates).To(HaveLen(1)) - update := mockStore.healthStatusUpdates[0] - Expect(update.Status).To(Equal(model.HealthStatusReady)) - Expect(update.ConsecutiveFailures).To(Equal(0)) - }) - }) - - Context("with a provider reporting unhealthy backing provider", func() { - It("transitions to Unhealthy immediately", func() { - server := unhealthyServer() - defer server.Close() - - providerID := uuid.New().String() - mockStore := &mockProviderStore{ - providers: model.ProviderList{ - { - ID: providerID, - Name: "test-provider", - Endpoint: server.URL, - HealthStatus: model.HealthStatusReady, - }, - }, - } - - monitor = healthcheck.NewMonitor(mockStore, &mockInstanceStore{}, cfg) - monitor.CheckProviders(ctx) - - Expect(mockStore.healthStatusUpdates).To(HaveLen(1)) - update := mockStore.healthStatusUpdates[0] - Expect(update.Status).To(Equal(model.HealthStatusUnhealthy)) - Expect(update.ConsecutiveFailures).To(Equal(0)) - }) - }) - - Context("with a failing provider", func() { - It("becomes Unavailable after reaching max consecutive failures", func() { - server := failingServer() - defer server.Close() - - providerID := uuid.New().String() - mockStore := &mockProviderStore{ - providers: model.ProviderList{ - { - ID: providerID, - Name: "test-provider", - Endpoint: server.URL, - HealthStatus: model.HealthStatusReady, - ConsecutiveFailures: 2, - }, - }, - } - - monitor = healthcheck.NewMonitor(mockStore, &mockInstanceStore{}, cfg) - monitor.CheckProviders(ctx) - - Expect(mockStore.healthStatusUpdates).To(HaveLen(1)) - update := mockStore.healthStatusUpdates[0] - Expect(update.Status).To(Equal(model.HealthStatusUnavailable)) - Expect(update.ConsecutiveFailures).To(Equal(3)) - }) - - It("stays Ready until reaching max consecutive failures", func() { - server := failingServer() - defer server.Close() - - providerID := uuid.New().String() - mockStore := &mockProviderStore{ - providers: model.ProviderList{ - { - ID: providerID, - Name: "test-provider", - Endpoint: server.URL, - HealthStatus: model.HealthStatusReady, - ConsecutiveFailures: 1, - }, - }, - } - - monitor = healthcheck.NewMonitor(mockStore, &mockInstanceStore{}, cfg) - monitor.CheckProviders(ctx) - - Expect(mockStore.healthStatusUpdates).To(HaveLen(1)) - update := mockStore.healthStatusUpdates[0] - Expect(update.Status).To(Equal(model.HealthStatusReady)) - Expect(update.ConsecutiveFailures).To(Equal(2)) - }) - }) - - Context("with a recovered provider", func() { - It("resets Unavailable provider to Ready with zero consecutive failures", func() { - server := healthyServer() - defer server.Close() - - providerID := uuid.New().String() - mockStore := &mockProviderStore{ - providers: model.ProviderList{ - { - ID: providerID, - Name: "test-provider", - Endpoint: server.URL, - HealthStatus: model.HealthStatusUnavailable, - ConsecutiveFailures: 5, - }, - }, - } - - monitor = healthcheck.NewMonitor(mockStore, &mockInstanceStore{}, cfg) - monitor.CheckProviders(ctx) - - Expect(mockStore.healthStatusUpdates).To(HaveLen(1)) - update := mockStore.healthStatusUpdates[0] - Expect(update.Status).To(Equal(model.HealthStatusReady)) - Expect(update.ConsecutiveFailures).To(Equal(0)) - }) - - It("resets Unhealthy provider to Ready with zero consecutive failures", func() { - server := healthyServer() - defer server.Close() - - providerID := uuid.New().String() - mockStore := &mockProviderStore{ - providers: model.ProviderList{ - { - ID: providerID, - Name: "test-provider", - Endpoint: server.URL, - HealthStatus: model.HealthStatusUnhealthy, - }, - }, - } - - monitor = healthcheck.NewMonitor(mockStore, &mockInstanceStore{}, cfg) - monitor.CheckProviders(ctx) - - Expect(mockStore.healthStatusUpdates).To(HaveLen(1)) - update := mockStore.healthStatusUpdates[0] - Expect(update.Status).To(Equal(model.HealthStatusReady)) - Expect(update.ConsecutiveFailures).To(Equal(0)) - }) - }) - - Context("deletion status transitions on provider health change", func() { - It("calls MarkProviderDeletionsPendingProvider when provider transitions Ready -> Unavailable", func() { - server := failingServer() - defer server.Close() - - providerID := uuid.New().String() - mockStore := &mockProviderStore{ - providers: model.ProviderList{ - { - ID: providerID, - Name: "failing-provider", - Endpoint: server.URL, - HealthStatus: model.HealthStatusReady, - ConsecutiveFailures: 2, - }, - }, - } - - instStore := &mockInstanceStore{} - monitor = healthcheck.NewMonitor(mockStore, instStore, cfg) - monitor.CheckProviders(ctx) - - Expect(instStore.markProviderDeletionsCalls).To(HaveLen(1)) - Expect(instStore.markProviderDeletionsCalls[0]).To(Equal("failing-provider")) - Expect(instStore.reactivateProviderCalls).To(BeEmpty()) - }) - - It("calls MarkProviderDeletionsPendingProvider when provider transitions Ready -> Unhealthy", func() { - server := unhealthyServer() - defer server.Close() - - providerID := uuid.New().String() - mockStore := &mockProviderStore{ - providers: model.ProviderList{ - { - ID: providerID, - Name: "unhealthy-provider", - Endpoint: server.URL, - HealthStatus: model.HealthStatusReady, - }, - }, - } - - instStore := &mockInstanceStore{} - monitor = healthcheck.NewMonitor(mockStore, instStore, cfg) - monitor.CheckProviders(ctx) - - Expect(instStore.markProviderDeletionsCalls).To(HaveLen(1)) - Expect(instStore.markProviderDeletionsCalls[0]).To(Equal("unhealthy-provider")) - Expect(instStore.reactivateProviderCalls).To(BeEmpty()) - }) - - It("calls ReactivateProviderDeletions when provider transitions Unavailable -> Ready", func() { - server := healthyServer() - defer server.Close() - - providerID := uuid.New().String() - mockStore := &mockProviderStore{ - providers: model.ProviderList{ - { - ID: providerID, - Name: "recovered-provider", - Endpoint: server.URL, - HealthStatus: model.HealthStatusUnavailable, - ConsecutiveFailures: 5, - }, - }, - } - - instStore := &mockInstanceStore{} - monitor = healthcheck.NewMonitor(mockStore, instStore, cfg) - monitor.CheckProviders(ctx) - - Expect(instStore.reactivateProviderCalls).To(HaveLen(1)) - Expect(instStore.reactivateProviderCalls[0]).To(Equal("recovered-provider")) - Expect(instStore.markProviderDeletionsCalls).To(BeEmpty()) - }) - - It("calls ReactivateProviderDeletions when provider transitions Unhealthy -> Ready", func() { - server := healthyServer() - defer server.Close() - - providerID := uuid.New().String() - mockStore := &mockProviderStore{ - providers: model.ProviderList{ - { - ID: providerID, - Name: "recovered-provider", - Endpoint: server.URL, - HealthStatus: model.HealthStatusUnhealthy, - }, - }, - } - - instStore := &mockInstanceStore{} - monitor = healthcheck.NewMonitor(mockStore, instStore, cfg) - monitor.CheckProviders(ctx) - - Expect(instStore.reactivateProviderCalls).To(HaveLen(1)) - Expect(instStore.reactivateProviderCalls[0]).To(Equal("recovered-provider")) - Expect(instStore.markProviderDeletionsCalls).To(BeEmpty()) - }) - - It("does not call either method when provider stays Ready", func() { - server := healthyServer() - defer server.Close() - - providerID := uuid.New().String() - mockStore := &mockProviderStore{ - providers: model.ProviderList{ - { - ID: providerID, - Name: "stable-provider", - Endpoint: server.URL, - HealthStatus: model.HealthStatusReady, - }, - }, - } - - instStore := &mockInstanceStore{} - monitor = healthcheck.NewMonitor(mockStore, instStore, cfg) - monitor.CheckProviders(ctx) - - Expect(instStore.markProviderDeletionsCalls).To(BeEmpty()) - Expect(instStore.reactivateProviderCalls).To(BeEmpty()) - }) - - It("does not call either method when provider stays Unavailable", func() { - server := failingServer() - defer server.Close() - - providerID := uuid.New().String() - mockStore := &mockProviderStore{ - providers: model.ProviderList{ - { - ID: providerID, - Name: "still-down-provider", - Endpoint: server.URL, - HealthStatus: model.HealthStatusUnavailable, - ConsecutiveFailures: 5, - }, - }, - } - - instStore := &mockInstanceStore{} - monitor = healthcheck.NewMonitor(mockStore, instStore, cfg) - monitor.CheckProviders(ctx) - - Expect(instStore.markProviderDeletionsCalls).To(BeEmpty()) - Expect(instStore.reactivateProviderCalls).To(BeEmpty()) - }) - - It("does not call either method when provider stays Unhealthy", func() { - server := unhealthyServer() - defer server.Close() - - providerID := uuid.New().String() - mockStore := &mockProviderStore{ - providers: model.ProviderList{ - { - ID: providerID, - Name: "still-unhealthy-provider", - Endpoint: server.URL, - HealthStatus: model.HealthStatusUnhealthy, - }, - }, - } - - instStore := &mockInstanceStore{} - monitor = healthcheck.NewMonitor(mockStore, instStore, cfg) - monitor.CheckProviders(ctx) - - Expect(instStore.markProviderDeletionsCalls).To(BeEmpty()) - Expect(instStore.reactivateProviderCalls).To(BeEmpty()) - }) - }) - }) -}) diff --git a/internal/sp/store/provider/provider_suite_test.go b/internal/sp/messaging/messaging_suite_test.go similarity index 54% rename from internal/sp/store/provider/provider_suite_test.go rename to internal/sp/messaging/messaging_suite_test.go index 48b78c5..93aeeb5 100644 --- a/internal/sp/store/provider/provider_suite_test.go +++ b/internal/sp/messaging/messaging_suite_test.go @@ -1,4 +1,4 @@ -package provider_test +package messaging_test import ( "testing" @@ -7,7 +7,7 @@ import ( . "github.com/onsi/gomega" ) -func TestProvider(t *testing.T) { +func TestMessaging(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Provider Store Suite") + RunSpecs(t, "Messaging Suite") } diff --git a/internal/sp/messaging/publisher.go b/internal/sp/messaging/publisher.go new file mode 100644 index 0000000..57c0c9a --- /dev/null +++ b/internal/sp/messaging/publisher.go @@ -0,0 +1,94 @@ +package messaging + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/cenkalti/backoff/v5" + "github.com/google/uuid" + "github.com/nats-io/nats.go/jetstream" +) + +type Publisher struct { + js jetstream.JetStream +} + +func NewPublisher(js jetstream.JetStream) *Publisher { + return &Publisher{js: js} +} + +// defaultPublishRetryOptions bounds retries to a few hundred milliseconds, +// since publish is called synchronously from request-handling paths. +// +// A fresh *backoff.ExponentialBackOff must be built on every call, not +// stored on the Publisher: it carries mutable state that Retry() mutates in +// place, and Publisher is shared across concurrent goroutines. +func defaultPublishRetryOptions() []backoff.RetryOption { + b := backoff.NewExponentialBackOff() + b.InitialInterval = 100 * time.Millisecond + b.MaxInterval = 1 * time.Second + b.Multiplier = 2.0 + return []backoff.RetryOption{ + backoff.WithBackOff(b), + backoff.WithMaxTries(4), // 1 initial attempt + 3 retries + } +} + +// EnsureStream creates or updates the agent request stream. Call during startup. +// WorkQueuePolicy is used because these are point-to-point work queues (one +// durable consumer per stream, each message consumed and acked exactly +// once) - messages are removed once acked instead of retained forever. +func (p *Publisher) EnsureStream(ctx context.Context) error { + _, err := p.js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ + Name: StreamName, + Subjects: []string{StreamSubjectBinding}, + Retention: jetstream.WorkQueuePolicy, + }) + if err != nil { + return fmt.Errorf("ensure agent request stream: %w", err) + } + return nil +} + +func (p *Publisher) PublishCreate(ctx context.Context, subject string, payload CreatePayload) error { + return p.publish(ctx, subject, CETypeCreateRequest, payload) +} + +func (p *Publisher) PublishDelete(ctx context.Context, subject string, payload DeletePayload) error { + return p.publish(ctx, subject, CETypeDeleteRequest, payload) +} + +func (p *Publisher) PublishCancel(ctx context.Context, subject string, payload CancelPayload) error { + return p.publish(ctx, subject, CETypeCancelRequest, payload) +} + +// publish marshals the CloudEvent envelope and publishes it with a bounded +// retry. The CE envelope "id" is also set as the NATS Nats-Msg-Id dedup +// header, so a retried publish (by this backoff loop, or by a caller like +// the pending sweep re-publishing after a timeout) that reaches JetStream +// twice within the dedup window is deduplicated server-side rather than +// producing a duplicate create/delete/cancel request to the agent. +func (p *Publisher) publish(ctx context.Context, subject, ceType string, payload any) error { + ceID := uuid.New().String() + envelope := map[string]any{ + "specversion": CESpecVersion, + "type": ceType, + "source": CESource, + "subject": subject, + "id": ceID, + "data": payload, + } + data, err := json.Marshal(envelope) + if err != nil { + return err + } + + operation := func() (struct{}, error) { + _, pubErr := p.js.Publish(ctx, subject, data, jetstream.WithMsgID(ceID)) + return struct{}{}, pubErr + } + _, err = backoff.Retry(ctx, operation, defaultPublishRetryOptions()...) + return err +} diff --git a/internal/sp/messaging/publisher_test.go b/internal/sp/messaging/publisher_test.go new file mode 100644 index 0000000..41b1d04 --- /dev/null +++ b/internal/sp/messaging/publisher_test.go @@ -0,0 +1,287 @@ +package messaging_test + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/dcm-project/control-plane/internal/sp/messaging" + natsserver "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// flakyJetStream fails the first N Publish calls with a transient-looking +// error, then delegates to the real JetStream so the publisher's retry +// logic (F9) can be exercised against a real stream/backend. +type flakyJetStream struct { + jetstream.JetStream + failures int + calls int +} + +func (f *flakyJetStream) Publish(ctx context.Context, subject string, data []byte, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + f.calls++ + if f.calls <= f.failures { + return nil, errors.New("simulated transient nats error") + } + return f.JetStream.Publish(ctx, subject, data, opts...) +} + +var _ = Describe("Publisher", func() { + var ( + ns *natsserver.Server + nc *nats.Conn + js jetstream.JetStream + publisher *messaging.Publisher + ctx context.Context + ) + + BeforeEach(func() { + opts := &natsserver.Options{ + Host: "127.0.0.1", + Port: -1, + JetStream: true, + StoreDir: GinkgoT().TempDir(), + } + var err error + ns, err = natsserver.NewServer(opts) + Expect(err).NotTo(HaveOccurred()) + ns.Start() + Expect(ns.ReadyForConnections(2 * time.Second)).To(BeTrue()) + + nc, err = nats.Connect(ns.ClientURL()) + Expect(err).NotTo(HaveOccurred()) + + js, err = jetstream.New(nc) + Expect(err).NotTo(HaveOccurred()) + + ctx = context.Background() + + _, err = js.CreateStream(ctx, jetstream.StreamConfig{ + Name: messaging.StreamName, + Subjects: []string{messaging.StreamSubjectBinding}, + }) + Expect(err).NotTo(HaveOccurred()) + + publisher = messaging.NewPublisher(js) + }) + + AfterEach(func() { + if nc != nil { + nc.Close() + } + if ns != nil { + ns.Shutdown() + } + }) + + fetchMessage := func(stream jetstream.Stream, seq uint64) jetstream.RawStreamMsg { + raw, err := stream.GetMsg(ctx, seq) + Expect(err).NotTo(HaveOccurred()) + return *raw + } + + Describe("Create", func() { + It("publishes to the agent's registered topic_name subject", func() { + agentTopic := "dcm.agent.prod-eu-west-1" + payload := messaging.CreatePayload{ + ResourceID: "res-1", + ServiceType: "vm", + Spec: map[string]any{"cpu": 4}, + } + + err := publisher.PublishCreate(ctx, agentTopic, payload) + Expect(err).NotTo(HaveOccurred()) + + stream, err := js.Stream(ctx, messaging.StreamName) + Expect(err).NotTo(HaveOccurred()) + + raw := fetchMessage(stream, 1) + Expect(raw.Subject).To(Equal(agentTopic)) + + var envelope map[string]any + Expect(json.Unmarshal(raw.Data, &envelope)).To(Succeed()) + Expect(envelope["type"]).To(Equal("dcm.request.create")) + Expect(envelope["source"]).To(Equal("dcm/control-plane")) + Expect(envelope["subject"]).To(Equal(agentTopic)) + Expect(envelope["specversion"]).To(Equal("1.0")) + + data := envelope["data"].(map[string]any) + Expect(data["resource_id"]).To(Equal("res-1")) + Expect(data["service_type"]).To(Equal("vm")) + Expect(data["spec"]).To(HaveKeyWithValue("cpu", float64(4))) + }) + }) + + Describe("Delete", func() { + It("publishes to the agent's registered topic_name subject (NOT .cancel)", func() { + agentTopic := "dcm.agent.prod-eu-west-1" + payload := messaging.DeletePayload{ + ResourceID: "res-1", + ServiceType: "vm", + } + + err := publisher.PublishDelete(ctx, agentTopic, payload) + Expect(err).NotTo(HaveOccurred()) + + stream, err := js.Stream(ctx, messaging.StreamName) + Expect(err).NotTo(HaveOccurred()) + + raw := fetchMessage(stream, 1) + Expect(raw.Subject).To(Equal(agentTopic)) + + var envelope map[string]any + Expect(json.Unmarshal(raw.Data, &envelope)).To(Succeed()) + Expect(envelope["type"]).To(Equal("dcm.request.delete")) + Expect(envelope["source"]).To(Equal("dcm/control-plane")) + Expect(envelope["subject"]).To(Equal(agentTopic)) + + data := envelope["data"].(map[string]any) + Expect(data["resource_id"]).To(Equal("res-1")) + Expect(data["service_type"]).To(Equal("vm")) + }) + }) + + Describe("Cancel", func() { + It("publishes to {topic_name}.cancel subject", func() { + agentTopic := "dcm.agent.prod-eu-west-1" + cancelSubject := agentTopic + ".cancel" + payload := messaging.CancelPayload{ + ResourceID: "res-1", + ServiceType: "vm", + } + + err := publisher.PublishCancel(ctx, cancelSubject, payload) + Expect(err).NotTo(HaveOccurred()) + + stream, err := js.Stream(ctx, messaging.StreamName) + Expect(err).NotTo(HaveOccurred()) + + raw := fetchMessage(stream, 1) + Expect(raw.Subject).To(Equal(cancelSubject)) + + var envelope map[string]any + Expect(json.Unmarshal(raw.Data, &envelope)).To(Succeed()) + Expect(envelope["type"]).To(Equal("dcm.request.cancel")) + Expect(envelope["source"]).To(Equal("dcm/control-plane")) + Expect(envelope["subject"]).To(Equal(cancelSubject)) + + data := envelope["data"].(map[string]any) + Expect(data["resource_id"]).To(Equal("res-1")) + Expect(data["service_type"]).To(Equal("vm")) + }) + + It("cancel subject must end with .cancel suffix", func() { + agentTopic := "dcm.agent.my-custom-topic" + cancelSubject := agentTopic + ".cancel" + payload := messaging.CancelPayload{ResourceID: "res-2", ServiceType: "container"} + + err := publisher.PublishCancel(ctx, cancelSubject, payload) + Expect(err).NotTo(HaveOccurred()) + + stream, err := js.Stream(ctx, messaging.StreamName) + Expect(err).NotTo(HaveOccurred()) + + raw := fetchMessage(stream, 1) + Expect(raw.Subject).To(HaveSuffix(".cancel")) + }) + }) + + Describe("Nats-Msg-Id dedup header", func() { + It("sets Nats-Msg-Id to the same value as the CloudEvent id", func() { + agentTopic := "dcm.agent.prod-eu-west-1" + payload := messaging.CreatePayload{ResourceID: "res-1", ServiceType: "vm", Spec: map[string]any{}} + + err := publisher.PublishCreate(ctx, agentTopic, payload) + Expect(err).NotTo(HaveOccurred()) + + stream, err := js.Stream(ctx, messaging.StreamName) + Expect(err).NotTo(HaveOccurred()) + raw := fetchMessage(stream, 1) + + var envelope map[string]any + Expect(json.Unmarshal(raw.Data, &envelope)).To(Succeed()) + ceID, ok := envelope["id"].(string) + Expect(ok).To(BeTrue()) + Expect(ceID).NotTo(BeEmpty()) + + Expect(raw.Header.Get("Nats-Msg-Id")).To(Equal(ceID)) + }) + }) + + Describe("Publish retry (transient failures)", func() { + It("retries on transient publish errors and eventually succeeds", func() { + flaky := &flakyJetStream{JetStream: js, failures: 2} + retryingPublisher := messaging.NewPublisher(flaky) + + agentTopic := "dcm.agent.retry-target" + payload := messaging.CreatePayload{ResourceID: "res-retry", ServiceType: "vm", Spec: map[string]any{}} + + err := retryingPublisher.PublishCreate(ctx, agentTopic, payload) + Expect(err).NotTo(HaveOccurred()) + Expect(flaky.calls).To(Equal(3)) + + stream, err := js.Stream(ctx, messaging.StreamName) + Expect(err).NotTo(HaveOccurred()) + raw := fetchMessage(stream, 1) + Expect(raw.Subject).To(Equal(agentTopic)) + }) + + It("returns an error once retries are exhausted", func() { + flaky := &flakyJetStream{JetStream: js, failures: 99} + retryingPublisher := messaging.NewPublisher(flaky) + + err := retryingPublisher.PublishCreate(ctx, "dcm.agent.always-fails", messaging.CreatePayload{ + ResourceID: "res-fail", ServiceType: "vm", Spec: map[string]any{}, + }) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("Stream retention", func() { + It("EnsureStream configures the agent request stream with WorkQueuePolicy", func() { + // BeforeEach already created the stream with default (limits) + // retention; JetStream disallows changing retention policy on an + // existing stream, so start from a clean slate for this test. + Expect(js.DeleteStream(ctx, messaging.StreamName)).To(Succeed()) + + Expect(publisher.EnsureStream(ctx)).To(Succeed()) + + stream, err := js.Stream(ctx, messaging.StreamName) + Expect(err).NotTo(HaveOccurred()) + info, err := stream.Info(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Config.Retention).To(Equal(jetstream.WorkQueuePolicy)) + }) + }) + + Describe("Topic routing uses registered topic_name, not agent name", func() { + It("different agents with different topic_names route correctly", func() { + topicA := "dcm.agent.custom-alpha" + topicB := "dcm.agent.custom-beta" + + err := publisher.PublishCreate(ctx, topicA, messaging.CreatePayload{ + ResourceID: "res-a", ServiceType: "vm", Spec: map[string]any{}, + }) + Expect(err).NotTo(HaveOccurred()) + + err = publisher.PublishCreate(ctx, topicB, messaging.CreatePayload{ + ResourceID: "res-b", ServiceType: "container", Spec: map[string]any{}, + }) + Expect(err).NotTo(HaveOccurred()) + + stream, err := js.Stream(ctx, messaging.StreamName) + Expect(err).NotTo(HaveOccurred()) + + rawA := fetchMessage(stream, 1) + Expect(rawA.Subject).To(Equal(topicA)) + + rawB := fetchMessage(stream, 2) + Expect(rawB.Subject).To(Equal(topicB)) + }) + }) +}) diff --git a/internal/sp/messaging/types.go b/internal/sp/messaging/types.go new file mode 100644 index 0000000..b9e0308 --- /dev/null +++ b/internal/sp/messaging/types.go @@ -0,0 +1,52 @@ +// Package messaging provides CloudEvent types and a JetStream publisher for agent communication. +package messaging + +const ( + CETypeCreateRequest = "dcm.request.create" + CETypeDeleteRequest = "dcm.request.delete" + CETypeCancelRequest = "dcm.request.cancel" + CESource = "dcm/control-plane" + CESpecVersion = "1.0" + StreamName = "dcm-agent-requests" + StreamSubjectBinding = "dcm.agent.>" +) + +// Agent response CE types. These are emitted by the agent (not the control +// plane) on the responses subject/stream; kept here as constants - alongside +// the request types above - so the consumer switch in +// internal/sp/consumer/response_consumer.go can't silently drift from what +// the agent actually sends. +const ( + CETypeCreationAcknowledged = "dcm.agent.creation-acknowledged" + CETypeError = "dcm.agent.error" + CETypeRequestQueued = "dcm.agent.request-queued" + CETypeDeletionAcknowledged = "dcm.agent.deletion-acknowledged" + CETypeCancelAcknowledged = "dcm.agent.cancel-acknowledged" + CETypeCancelRejected = "dcm.agent.cancel-rejected" +) + +// ResponseStreamName/ResponseSubject are the wire contract for the agent +// response stream, consumed by consumer.ResponseConsumer. Exported here +// (rather than kept private to that package) so other producers of these +// events - real agents, and tests simulating one - publish to the same +// subject the consumer actually listens on. +const ( + ResponseStreamName = "dcm-agent-responses" + ResponseSubject = "dcm.agents.responses" +) + +type CreatePayload struct { + ResourceID string `json:"resource_id"` + ServiceType string `json:"service_type"` + Spec map[string]any `json:"spec"` +} + +type DeletePayload struct { + ResourceID string `json:"resource_id"` + ServiceType string `json:"service_type"` +} + +type CancelPayload struct { + ResourceID string `json:"resource_id"` + ServiceType string `json:"service_type"` +} diff --git a/internal/sp/pending/pending_suite_test.go b/internal/sp/pending/pending_suite_test.go new file mode 100644 index 0000000..ee138a4 --- /dev/null +++ b/internal/sp/pending/pending_suite_test.go @@ -0,0 +1,13 @@ +package pending_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestPending(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Pending Sweep Suite") +} diff --git a/internal/sp/pending/sweep.go b/internal/sp/pending/sweep.go new file mode 100644 index 0000000..f6b43c8 --- /dev/null +++ b/internal/sp/pending/sweep.go @@ -0,0 +1,353 @@ +// Package pending sweeps timed-out pending and queued service instances. +package pending + +import ( + "context" + "errors" + "log/slog" + "sync" + "time" + + agentstore "github.com/dcm-project/control-plane/internal/agent/store/agent" + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" + "github.com/dcm-project/control-plane/internal/sp/messaging" + "github.com/dcm-project/control-plane/internal/sp/store/model" + "gorm.io/gorm" +) + +var errAgentNotReady = errors.New("agent not ready") + +// Reevaluator re-routes an existing resource to a different agent, excluding +// the given agent names, and re-triggers provisioning. Implemented by +// placement's PlacementService; this interface keeps the sp/pending package +// free of a compile-time dependency on the placement domain. +type Reevaluator interface { + ReEvaluateWithExclude(ctx context.Context, resourceID string, excludeAgents []string) error +} + +type Sweep struct { + db *gorm.DB + publisher *messaging.Publisher + agentStore agentstore.Agent + reevaluator Reevaluator + pendingTimeout time.Duration + queuedTimeout time.Duration + maxRetries int + interval time.Duration + stopCh chan struct{} + stopOnce sync.Once + wg sync.WaitGroup +} + +func NewSweep(db *gorm.DB, publisher *messaging.Publisher, agentSt agentstore.Agent, reevaluator Reevaluator, pendingTimeout, queuedTimeout, interval time.Duration, maxRetries int) *Sweep { + return &Sweep{ + db: db, + publisher: publisher, + agentStore: agentSt, + reevaluator: reevaluator, + pendingTimeout: pendingTimeout, + queuedTimeout: queuedTimeout, + maxRetries: maxRetries, + interval: interval, + stopCh: make(chan struct{}), + } +} + +func (s *Sweep) Start(ctx context.Context) { + ctx, cancel := context.WithCancel(ctx) + s.wg.Add(1) + go func() { + defer s.wg.Done() + defer cancel() + s.sweep(ctx) + ticker := time.NewTicker(s.interval) + defer ticker.Stop() + for { + select { + case <-s.stopCh: + return + case <-ctx.Done(): + return + case <-ticker.C: + s.sweep(ctx) + } + } + }() +} + +func (s *Sweep) Stop() { + s.stopOnce.Do(func() { close(s.stopCh) }) + s.wg.Wait() +} + +func (s *Sweep) sweep(ctx context.Context) { + s.sweepPending(ctx) + s.sweepQueued(ctx) +} + +func (s *Sweep) sweepPending(ctx context.Context) { + cutoff := time.Now().Add(-s.pendingTimeout) + var instances []model.ServiceTypeInstance + // deletion_status filter mirrors sweepQueued: a deferred DeleteInstance + // never touches Status, so a "pending" instance can have a delete + // already SCHEDULED against it and must not be picked up for self-healing. + err := s.db.WithContext(ctx).Where("status = ? AND pending_started_at < ? AND agent_name IS NOT NULL AND (deletion_status IS NULL OR deletion_status = '')", model.StatusPending, cutoff). + Find(&instances).Error + if err != nil { + slog.Error("sweep: failed to query pending instances", "error", err) + return + } + + for i := range instances { + s.retryPendingInstance(ctx, &instances[i], cutoff) + } +} + +// retryPendingInstance implements the self-healing loop for a timed-out +// pending instance: instead of blindly re-publishing to the same agent that +// already failed to pick it up, it asks the placement layer to re-evaluate +// policy excluding that agent and re-provision against a different one. +// Only when no alternate agent is available (either because retries are +// exhausted or re-evaluation itself fails) does the instance get marked +// "failed". +func (s *Sweep) retryPendingInstance(ctx context.Context, inst *model.ServiceTypeInstance, cutoff time.Time) { + log := slog.With("instance_id", inst.ID) + + if s.reevaluator == nil { + log.Warn("sweep: no reevaluator configured, cannot self-heal pending instance") + return + } + + if inst.RetryCount >= s.maxRetries { + // Still claim (same CAS as the non-exhausted path below) before the + // final attempt: without it, two control-plane replicas could both + // read this instance past maxRetries and both call selfHeal + // concurrently for the "final attempt", double-provisioning it. + claimed, err := s.claimRetry(ctx, inst.ID, cutoff) + if err != nil { + log.Error("sweep: failed to claim final retry attempt", "error", err) + return + } + if !claimed { + log.Debug("sweep: final retry attempt already claimed by another sweep") + return + } + if err := s.selfHeal(ctx, inst); err == nil { + log.Info("sweep: found alternate agent on final attempt, instance resumed") + return + } + applied, err := s.markFailedFrom(ctx, inst.ID, model.StatusPending) + if err != nil { + log.Error("sweep: failed to mark instance as failed", "error", err) + return + } + if !applied { + log.Debug("sweep: instance already moved on before it could be marked failed") + return + } + log.Info("sweep: pending instance retries exhausted, no viable agent found") + return + } + + claimed, err := s.claimRetry(ctx, inst.ID, cutoff) + if err != nil { + log.Error("sweep: failed to claim retry", "error", err) + return + } + if !claimed { + log.Debug("sweep: instance already claimed by another sweep") + return + } + + if err := s.selfHeal(ctx, inst); err != nil { + log.Warn("sweep: re-evaluation to a new agent failed, will retry next cycle", "error", err) + return + } + log.Info("sweep: pending instance re-routed to a new agent") +} + +// selfHeal asks the placement layer to pick a new agent (excluding the +// instance's current agent, if any) and re-provision against it. +func (s *Sweep) selfHeal(ctx context.Context, inst *model.ServiceTypeInstance) error { + var excludeAgents []string + if inst.AgentName != nil { + excludeAgents = []string{*inst.AgentName} + } + return s.reevaluator.ReEvaluateWithExclude(ctx, inst.ID, excludeAgents) +} + +// markFailedFrom CAS-guards the terminal "failed" transition on fromStatus, +// so a concurrent change elsewhere can't be silently clobbered. Returns +// applied=false (no error) if the instance had already moved off fromStatus. +func (s *Sweep) markFailedFrom(ctx context.Context, id string, fromStatus string) (bool, error) { + result := s.db.WithContext(ctx).Model(&model.ServiceTypeInstance{}). + Where("id = ? AND status = ?", id, fromStatus). + Updates(map[string]any{"status": model.StatusFailed, "status_message": "retries exhausted"}) + if result.Error != nil { + return false, result.Error + } + return result.RowsAffected > 0, nil +} + +// markPendingFrom CAS-guards transitioning an instance back to "pending" +// (with a fresh pending_started_at) from fromStatus, so a failed self-heal +// attempt out of "cancelled" doesn't strand the instance in a status +// neither sweepPending nor sweepQueued ever revisits. +func (s *Sweep) markPendingFrom(ctx context.Context, id string, fromStatus string) (bool, error) { + result := s.db.WithContext(ctx).Model(&model.ServiceTypeInstance{}). + Where("id = ? AND status = ?", id, fromStatus). + Updates(map[string]any{"status": model.StatusPending, "pending_started_at": time.Now()}) + if result.Error != nil { + return false, result.Error + } + return result.RowsAffected > 0, nil +} + +// claimRetry atomically increments retry_count and resets the pending timer. +// The cutoff check (matching sweepPending's own cutoff) closes a race where +// two sweep replicas read the same stale instance: the second claim's WHERE +// clause fails against the fresh pending_started_at the first just wrote. +func (s *Sweep) claimRetry(ctx context.Context, id string, cutoff time.Time) (bool, error) { + result := s.db.WithContext(ctx).Model(&model.ServiceTypeInstance{}). + Where("id = ? AND status = ? AND pending_started_at < ?", id, model.StatusPending, cutoff). + Updates(map[string]any{ + "retry_count": gorm.Expr("retry_count + 1"), + "pending_started_at": time.Now(), + }) + if result.Error != nil { + return false, result.Error + } + return result.RowsAffected > 0, nil +} + +func (s *Sweep) sweepQueued(ctx context.Context) { + cutoff := time.Now().Add(-s.queuedTimeout) + var instances []model.ServiceTypeInstance + err := s.db.WithContext(ctx).Where("status = ? AND pending_started_at < ? AND agent_name IS NOT NULL AND (deletion_status IS NULL OR deletion_status = '')", model.StatusQueued, cutoff). + Find(&instances).Error + if err != nil { + slog.Error("sweep: failed to query queued instances", "error", err) + return + } + + for i := range instances { + s.cancelQueuedInstance(ctx, &instances[i], cutoff) + } +} + +// cancelQueuedInstance handles a queued instance whose agent never +// acknowledged the create request in time: claims it via CAS, best-effort +// notifies the old agent to stand down, then self-heals to a different +// agent (or gives up once the shared retry budget is exhausted). +// "cancelled" is a transient bookkeeping state on the way to a fresh +// pending assignment, not a terminal outcome. +func (s *Sweep) cancelQueuedInstance(ctx context.Context, inst *model.ServiceTypeInstance, cutoff time.Time) { + log := slog.With("instance_id", inst.ID) + + claimed, err := s.claimCancel(ctx, inst.ID, cutoff) + if err != nil { + log.Error("sweep: failed to cancel queued instance", "error", err) + return + } + if !claimed { + log.Debug("sweep: queued instance already moved by response consumer or another sweep") + return + } + + s.notifyAgentOfCancel(ctx, inst) + + if s.reevaluator == nil { + return + } + + if inst.RetryCount+1 >= s.maxRetries { + if err := s.selfHeal(ctx, inst); err == nil { + log.Info("sweep: found alternate agent on final attempt, queued instance resumed") + return + } + applied, err := s.markFailedFrom(ctx, inst.ID, model.StatusCancelled) + if err != nil { + log.Error("sweep: failed to mark instance as failed", "error", err) + return + } + if !applied { + log.Debug("sweep: instance already moved on before it could be marked failed") + return + } + log.Info("sweep: queued instance retries exhausted, no viable agent found") + return + } + + if err := s.selfHeal(ctx, inst); err != nil { + // Fall back to "pending" instead of leaving the instance stranded + // in "cancelled": neither sweepPending nor sweepQueued ever + // revisits that status, so the next sweepPending cycle retries it. + applied, markErr := s.markPendingFrom(ctx, inst.ID, model.StatusCancelled) + if markErr != nil { + log.Error("sweep: failed to fall back cancelled instance to pending for retry", "error", markErr) + return + } + if !applied { + log.Debug("sweep: instance already moved on before it could fall back to pending", "self_heal_error", err) + return + } + log.Info("sweep: no alternate agent available for cancelled instance, reverted to pending for next retry", "error", err) + return + } + log.Info("sweep: cancelled instance re-routed to a new agent") +} + +// claimCancel CAS-transitions a timed-out queued instance to "cancelled" and +// claims a retry, guarded by the same cutoff sweepQueued used to select it - +// so a horizontally-scaled control plane can't double-claim it, and it +// can't race the response consumer's own status-CAS. +func (s *Sweep) claimCancel(ctx context.Context, id string, cutoff time.Time) (bool, error) { + result := s.db.WithContext(ctx).Model(&model.ServiceTypeInstance{}). + Where("id = ? AND status = ? AND pending_started_at < ?", id, model.StatusQueued, cutoff). + Updates(map[string]any{ + "status": model.StatusCancelled, + "retry_count": gorm.Expr("retry_count + 1"), + }) + if result.Error != nil { + return false, result.Error + } + return result.RowsAffected > 0, nil +} + +// notifyAgentOfCancel best-effort tells the old agent to stand down. It's a +// courtesy, not a precondition for the self-heal that follows, and always +// runs after claimCancel has already won: notifying before the claim would +// let every replica racing for it send a duplicate cancel to the agent. +func (s *Sweep) notifyAgentOfCancel(ctx context.Context, inst *model.ServiceTypeInstance) { + if s.publisher == nil || s.agentStore == nil || inst.AgentName == nil { + return + } + log := slog.With("instance_id", inst.ID) + subject, err := s.resolveSubjectWithError(ctx, *inst.AgentName) + switch { + case err == nil: + if pubErr := s.publisher.PublishCancel(ctx, subject+".cancel", messaging.CancelPayload{ + ResourceID: inst.ID, + ServiceType: inst.ServiceType, + }); pubErr != nil { + log.Warn("sweep: cancel publish failed, proceeding to self-heal anyway", "error", pubErr) + } + case errors.Is(err, agentstore.ErrAgentNotFound): + log.Info("sweep: agent not found, cancelling locally", "agent_name", *inst.AgentName) + case errors.Is(err, errAgentNotReady): + log.Info("sweep: agent not ready, cancelling locally", "agent_name", *inst.AgentName) + default: + log.Warn("sweep: failed to resolve agent for cancel notification, proceeding to self-heal anyway", "agent_name", *inst.AgentName, "error", err) + } +} + +func (s *Sweep) resolveSubjectWithError(ctx context.Context, agentName string) (string, error) { + agent, err := s.agentStore.GetByName(ctx, agentName) + if err != nil { + return "", err + } + if agent.HealthStatus != agentmodel.AgentHealthStatusReady { + return "", errAgentNotReady + } + return agent.TopicName, nil +} diff --git a/internal/sp/pending/sweep_test.go b/internal/sp/pending/sweep_test.go new file mode 100644 index 0000000..98c5396 --- /dev/null +++ b/internal/sp/pending/sweep_test.go @@ -0,0 +1,513 @@ +package pending_test + +import ( + "context" + "fmt" + "sync" + "time" + + agentstore "github.com/dcm-project/control-plane/internal/agent/store/agent" + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" + "github.com/dcm-project/control-plane/internal/sp/messaging" + "github.com/dcm-project/control-plane/internal/sp/pending" + "github.com/dcm-project/control-plane/internal/sp/store/model" + "github.com/google/uuid" + "github.com/nats-io/nats.go/jetstream" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +type failingJetStream struct { + jetstream.JetStream + publishErr error +} + +func (m *failingJetStream) Publish(_ context.Context, _ string, _ []byte, _ ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + return nil, m.publishErr +} + +// limitToSingleConn pins the pool to one physical connection. sqlite's +// ":memory:" DSN gives each new physical connection its own empty database, +// so once the sweep's background goroutine and the test's own Eventually/ +// Consistently assertions query concurrently, a second pooled connection +// would see "no such table" instead of the migrated schema. +func limitToSingleConn(d *gorm.DB) { + sqlDB, err := d.DB() + Expect(err).NotTo(HaveOccurred()) + sqlDB.SetMaxOpenConns(1) +} + +// fakeReevaluator records ReEvaluateWithExclude invocations so tests can +// assert the self-healing loop actually calls into placement with the right +// arguments, without depending on the real policy/SPRM stack. +type fakeReevaluator struct { + mu sync.Mutex + calls []reevalCall + err error +} + +type reevalCall struct { + resourceID string + excludeAgents []string +} + +func (f *fakeReevaluator) ReEvaluateWithExclude(_ context.Context, resourceID string, excludeAgents []string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, reevalCall{resourceID: resourceID, excludeAgents: excludeAgents}) + return f.err +} + +func (f *fakeReevaluator) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.calls) +} + +func (f *fakeReevaluator) lastCall() reevalCall { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls[len(f.calls)-1] +} + +var _ = Describe("Pending Sweep", func() { + var ( + db *gorm.DB + sweep *pending.Sweep + ctx context.Context + ) + + AfterEach(func() { + sweep.Stop() + sqlDB, _ := db.DB() + _ = sqlDB.Close() + }) + + newDB := func() *gorm.DB { + d, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + Expect(err).NotTo(HaveOccurred()) + limitToSingleConn(d) + Expect(d.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) + Expect(d.Create(&agentmodel.Agent{ + ID: uuid.New().String(), Name: "test-agent", + TopicName: "dcm.agent.test-agent", HealthStatus: agentmodel.AgentHealthStatusReady, + }).Error).NotTo(HaveOccurred()) + return d + } + + pendingInstance := func(retryCount int) model.ServiceTypeInstance { + pastTime := time.Now().Add(-1 * time.Minute) + agentName := "test-agent" + return model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", Status: "pending", InstanceName: "sweep-test", + Spec: map[string]any{"cpu": 2}, AgentName: &agentName, + PendingStartedAt: &pastTime, RetryCount: retryCount, + } + } + + It("does not consume retries when reevaluator is nil", func() { + db = newDB() + sweep = pending.NewSweep(db, nil, nil, nil, 30*time.Second, 60*time.Second, 5*time.Millisecond, 3) + ctx = context.Background() + + inst := pendingInstance(0) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Consistently(func() int { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + return updated.RetryCount + }, 200*time.Millisecond, 20*time.Millisecond).Should(Equal(0)) + + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + Expect(updated.Status).To(Equal("pending")) + }) + + It("re-routes to a new agent via reevaluator when retries remain", func() { + db = newDB() + reeval := &fakeReevaluator{} + sweep = pending.NewSweep(db, nil, nil, reeval, 30*time.Second, 60*time.Second, 5*time.Millisecond, 3) + ctx = context.Background() + + inst := pendingInstance(0) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Eventually(reeval.callCount, time.Second, 10*time.Millisecond).Should(BeNumerically(">=", 1)) + call := reeval.lastCall() + Expect(call.resourceID).To(Equal(inst.ID)) + Expect(call.excludeAgents).To(ConsistOf("test-agent")) + + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + Expect(updated.Status).To(Equal("pending")) + Expect(updated.RetryCount).To(BeNumerically(">=", 1)) + }) + + It("keeps instance pending and retries again when reevaluation fails", func() { + db = newDB() + reeval := &fakeReevaluator{err: fmt.Errorf("no agent available")} + sweep = pending.NewSweep(db, nil, nil, reeval, 30*time.Second, 60*time.Second, 5*time.Millisecond, 3) + ctx = context.Background() + + inst := pendingInstance(0) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Eventually(reeval.callCount, time.Second, 10*time.Millisecond).Should(BeNumerically(">=", 1)) + + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + Expect(updated.Status).To(Equal("pending")) + }) + + It("marks as failed when retries exhausted and no alternate agent exists", func() { + db = newDB() + reeval := &fakeReevaluator{err: fmt.Errorf("no agent available")} + sweep = pending.NewSweep(db, nil, nil, reeval, 30*time.Second, 60*time.Second, 5*time.Millisecond, 3) + ctx = context.Background() + + inst := pendingInstance(3) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Eventually(func() string { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + return updated.Status + }, time.Second, 10*time.Millisecond).Should(Equal("failed")) + + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + Expect(updated.StatusMessage).To(Equal("retries exhausted")) + }) + + It("resumes on the final attempt when reevaluation finds an alternate agent", func() { + db = newDB() + reeval := &fakeReevaluator{} + sweep = pending.NewSweep(db, nil, nil, reeval, 30*time.Second, 60*time.Second, 5*time.Millisecond, 3) + ctx = context.Background() + + inst := pendingInstance(3) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Eventually(reeval.callCount, time.Second, 10*time.Millisecond).Should(BeNumerically(">=", 1)) + + Consistently(func() string { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + return updated.Status + }, 200*time.Millisecond, 20*time.Millisecond).ShouldNot(Equal("failed")) + }) + + It("never self-heals a pending instance that already has a delete scheduled against it (R2 S2: finding #1)", func() { + // A deferred DeleteInstance never touches Status, so a timed-out + // "pending" instance can have a delete already SCHEDULED. + // sweepPending must exclude it, like sweepQueued already does. + db = newDB() + reeval := &fakeReevaluator{} + sweep = pending.NewSweep(db, nil, nil, reeval, 30*time.Second, 60*time.Second, 5*time.Millisecond, 3) + ctx = context.Background() + + inst := pendingInstance(0) + deletionStatus := "SCHEDULED" + inst.DeletionStatus = &deletionStatus + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Consistently(func() int { + return reeval.callCount() + }, 300*time.Millisecond, 20*time.Millisecond).Should(Equal(0)) + + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + Expect(updated.Status).To(Equal("pending")) + Expect(updated.AgentName).NotTo(BeNil()) + Expect(*updated.AgentName).To(Equal("test-agent")) + }) +}) + +var _ = Describe("Queued Sweep", func() { + var ( + db *gorm.DB + sweep *pending.Sweep + ctx context.Context + ) + + BeforeEach(func() { + var err error + db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + Expect(err).NotTo(HaveOccurred()) + limitToSingleConn(db) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) + Expect(db.Create(&agentmodel.Agent{ID: uuid.New().String(), Name: "test-agent", TopicName: "dcm.agent.test-agent"}).Error).NotTo(HaveOccurred()) + + sweep = pending.NewSweep(db, nil, nil, nil, 30*time.Second, 60*time.Second, 5*time.Millisecond, 3) + ctx = context.Background() + }) + + AfterEach(func() { + sweep.Stop() + sqlDB, _ := db.DB() + _ = sqlDB.Close() + }) + + It("cancels request after queued timeout", func() { + pastTime := time.Now().Add(-2 * time.Minute) + agentName := "test-agent" + instance := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "queued", + InstanceName: "queued-sweep", + Spec: map[string]any{"cpu": 2}, + AgentName: &agentName, + PendingStartedAt: &pastTime, + } + Expect(db.Create(&instance).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Eventually(func() string { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", instance.ID).Error).NotTo(HaveOccurred()) + return updated.Status + }, time.Second, 10*time.Millisecond).Should(Equal("cancelled")) + }) + + It("skips deletion requests", func() { + pastTime := time.Now().Add(-2 * time.Minute) + agentName := "test-agent" + deletionStatus := "pending_deletion" + instance := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "queued", + InstanceName: "queued-delete", + Spec: map[string]any{"cpu": 2}, + AgentName: &agentName, + PendingStartedAt: &pastTime, + DeletionStatus: &deletionStatus, + } + Expect(db.Create(&instance).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Consistently(func() string { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", instance.ID).Error).NotTo(HaveOccurred()) + return updated.Status + }, 200*time.Millisecond, 20*time.Millisecond).Should(Equal("queued")) + }) +}) + +var _ = Describe("Queued Sweep Cancellation", func() { + var ( + db *gorm.DB + sweep *pending.Sweep + ctx context.Context + ) + + AfterEach(func() { + sweep.Stop() + sqlDB, _ := db.DB() + _ = sqlDB.Close() + }) + + queuedInstance := func(agentName string) model.ServiceTypeInstance { + pastTime := time.Now().Add(-2 * time.Minute) + return model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", Status: "queued", InstanceName: "queued-cancel-test", + Spec: map[string]any{"cpu": 2}, AgentName: &agentName, + PendingStartedAt: &pastTime, + } + } + + It("still cancels (claims) a queued instance even when the best-effort agent-cancel publish fails", func() { + // The CAS claim runs BEFORE the publish attempt (see + // cancelQueuedInstance): a failure to notify the old agent is a + // courtesy best-effort, not a precondition for committing to move + // away from it, so the instance still transitions to "cancelled" + // even though the publish below always fails. + var err error + db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + Expect(err).NotTo(HaveOccurred()) + limitToSingleConn(db) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) + Expect(db.Create(&agentmodel.Agent{ + ID: uuid.New().String(), Name: "test-agent", + TopicName: "dcm.agent.test-agent", HealthStatus: agentmodel.AgentHealthStatusReady, + }).Error).NotTo(HaveOccurred()) + + js := &failingJetStream{publishErr: fmt.Errorf("nats: connection closed")} + pub := messaging.NewPublisher(js) + agentSt := agentstore.NewAgent(db) + // reevaluator is nil, so once the CAS claims the instance the sweep + // stops there (no self-heal attempted) - isolating this test to just + // the claim-vs-publish ordering. + sweep = pending.NewSweep(db, pub, agentSt, nil, 30*time.Second, 60*time.Second, 5*time.Millisecond, 3) + ctx = context.Background() + + inst := queuedInstance("test-agent") + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Eventually(func() string { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + return updated.Status + }, time.Second, 10*time.Millisecond).Should(Equal("cancelled")) + }) + + It("cancels locally when agent is not found", func() { + var err error + db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + Expect(err).NotTo(HaveOccurred()) + limitToSingleConn(db) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) + + js := &failingJetStream{publishErr: fmt.Errorf("should not be called")} + pub := messaging.NewPublisher(js) + agentSt := agentstore.NewAgent(db) + sweep = pending.NewSweep(db, pub, agentSt, nil, 30*time.Second, 60*time.Second, 5*time.Millisecond, 3) + ctx = context.Background() + + inst := queuedInstance("nonexistent-agent") + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Eventually(func() string { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + return updated.Status + }, time.Second, 10*time.Millisecond).Should(Equal("cancelled")) + }) + + It("cancels locally when agent is unavailable", func() { + var err error + db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + Expect(err).NotTo(HaveOccurred()) + limitToSingleConn(db) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) + Expect(db.Create(&agentmodel.Agent{ + ID: uuid.New().String(), Name: "test-agent", + TopicName: "dcm.agent.test-agent", HealthStatus: agentmodel.AgentHealthStatusUnavailable, + }).Error).NotTo(HaveOccurred()) + + js := &failingJetStream{publishErr: fmt.Errorf("should not be called")} + pub := messaging.NewPublisher(js) + agentSt := agentstore.NewAgent(db) + sweep = pending.NewSweep(db, pub, agentSt, nil, 30*time.Second, 60*time.Second, 5*time.Millisecond, 3) + ctx = context.Background() + + inst := queuedInstance("test-agent") + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Eventually(func() string { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + return updated.Status + }, time.Second, 10*time.Millisecond).Should(Equal("cancelled")) + }) + + It("marks a queued instance failed once retries are exhausted instead of bouncing forever", func() { + var err error + db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + Expect(err).NotTo(HaveOccurred()) + limitToSingleConn(db) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) + + reeval := &fakeReevaluator{err: fmt.Errorf("no agent available")} + sweep = pending.NewSweep(db, nil, nil, reeval, 30*time.Second, 60*time.Second, 5*time.Millisecond, 3) + ctx = context.Background() + + inst := queuedInstance("nonexistent-agent") + inst.RetryCount = 3 + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Eventually(func() string { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + return updated.Status + }, time.Second, 10*time.Millisecond).Should(Equal("failed")) + }) + + It("self-heals a cancelled instance when the reevaluator finds an alternate agent", func() { + var err error + db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + Expect(err).NotTo(HaveOccurred()) + limitToSingleConn(db) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) + + reeval := &fakeReevaluator{} + sweep = pending.NewSweep(db, nil, nil, reeval, 30*time.Second, 60*time.Second, 5*time.Millisecond, 3) + ctx = context.Background() + + inst := queuedInstance("nonexistent-agent") + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Eventually(reeval.callCount, time.Second, 10*time.Millisecond).Should(BeNumerically(">=", 1)) + call := reeval.lastCall() + Expect(call.resourceID).To(Equal(inst.ID)) + Expect(call.excludeAgents).To(ConsistOf("nonexistent-agent")) + }) + + It("falls back a cancelled instance to pending for a later retry instead of stranding it, when self-heal fails with retries left (R2 S3: finding #4)", func() { + // Neither sweepPending (selects "pending") nor sweepQueued (selects + // "queued") ever revisits a "cancelled" row, so if the immediate + // self-heal attempt fails while retry budget remains, the instance + // must fall back to "pending" (picked up again by sweepPending next + // cycle) rather than staying "cancelled" forever. + var err error + db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + Expect(err).NotTo(HaveOccurred()) + limitToSingleConn(db) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) + + reeval := &fakeReevaluator{err: fmt.Errorf("no agent available")} + sweep = pending.NewSweep(db, nil, nil, reeval, 30*time.Second, 60*time.Second, 5*time.Millisecond, 3) + ctx = context.Background() + + inst := queuedInstance("nonexistent-agent") + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + sweep.Start(ctx) + + Eventually(func() string { + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + return updated.Status + }, time.Second, 10*time.Millisecond).Should(Equal("pending")) + + var updated model.ServiceTypeInstance + Expect(db.First(&updated, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + Expect(updated.RetryCount).To(Equal(1)) + Expect(updated.PendingStartedAt).NotTo(BeNil()) + }) +}) diff --git a/internal/sp/service/errors.go b/internal/sp/service/errors.go index e7324a5..9d64b0f 100644 --- a/internal/sp/service/errors.go +++ b/internal/sp/service/errors.go @@ -5,11 +5,12 @@ import "errors" // Error codes returned by service operations. const ( - ErrCodeNotFound = "https://dcm.example.com/errors/not-found" - ErrCodeConflict = "https://dcm.example.com/errors/conflict" - ErrCodeValidation = "https://dcm.example.com/errors/validation" - ErrCodeProviderError = "https://dcm.example.com/errors/provider-error" - ErrCodeInternal = "https://dcm.example.com/errors/internal-error" + ErrCodeNotFound = "https://dcm.example.com/errors/not-found" + ErrCodeConflict = "https://dcm.example.com/errors/conflict" + ErrCodeValidation = "https://dcm.example.com/errors/validation" + ErrCodeProvisioningError = "https://dcm.example.com/errors/provisioning-error" + ErrCodeInternal = "https://dcm.example.com/errors/internal-error" + ErrCodeUnavailable = "https://dcm.example.com/errors/unavailable" ) // ServiceError represents a business logic error with a code for HTTP mapping. @@ -45,9 +46,9 @@ func NewValidationError(message string) *ServiceError { } } -func NewProviderError(message string) *ServiceError { +func NewProvisioningError(message string) *ServiceError { return &ServiceError{ - Code: ErrCodeProviderError, + Code: ErrCodeProvisioningError, Message: message, } } @@ -59,6 +60,13 @@ func NewInternalError(message string) *ServiceError { } } +func NewUnavailableError(message string) *ServiceError { + return &ServiceError{ + Code: ErrCodeUnavailable, + Message: message, + } +} + // IsClientError returns true if err is a ServiceError representing a client-side // (4xx) problem. If svcErr is non-nil it is populated with the unwrapped error. func IsClientError(err error, svcErr **ServiceError) bool { @@ -66,7 +74,7 @@ func IsClientError(err error, svcErr **ServiceError) bool { return false } switch (*svcErr).Code { - case ErrCodeValidation, ErrCodeNotFound, ErrCodeConflict, ErrCodeProviderError: + case ErrCodeValidation, ErrCodeNotFound, ErrCodeConflict, ErrCodeProvisioningError: return true } return false diff --git a/internal/sp/service/provider/convert.go b/internal/sp/service/provider/convert.go deleted file mode 100644 index 5056b69..0000000 --- a/internal/sp/service/provider/convert.go +++ /dev/null @@ -1,91 +0,0 @@ -package provider - -import ( - "encoding/json" - - providerserver "github.com/dcm-project/control-plane/internal/sp/api/provider" - "github.com/dcm-project/control-plane/internal/sp/service" - "github.com/dcm-project/control-plane/internal/sp/store/model" -) - -// ModelToProvider converts a database model to an API response type. -func ModelToProvider(m *model.Provider) *providerserver.Provider { - p := &providerserver.Provider{ - Id: &m.ID, - Name: m.Name, - ServiceType: m.ServiceType, - SchemaVersion: m.SchemaVersion, - Endpoint: m.Endpoint, - DisplayName: m.DisplayName, - HealthStatus: m.HealthStatus.StringPtr(), - CreateTime: service.PtrTime(m.CreateTime), - UpdateTime: service.PtrTime(m.UpdateTime), - } - if m.Metadata != nil { - b, err := json.Marshal(m.Metadata) - if err == nil { - var meta providerserver.ProviderMetadata - if err := json.Unmarshal(b, &meta); err == nil { - p.Metadata = &meta - } - } - } - if opBytes, err := json.Marshal(m.Operations); err == nil && string(opBytes) != "null" { - var ops []string - if err := json.Unmarshal(opBytes, &ops); err == nil { - p.Operations = &ops - } - } - return p -} - -// ProviderToModel converts an API request to a database model. -func ProviderToModel(req *providerserver.Provider, id string) model.Provider { - m := model.Provider{ - ID: id, - Name: req.Name, - ServiceType: req.ServiceType, - SchemaVersion: req.SchemaVersion, - Endpoint: req.Endpoint, - DisplayName: req.DisplayName, - } - if req.Metadata != nil { - if metaMap, err := providerMetadataToMap(req.Metadata); err == nil { - m.Metadata = metaMap - } - } - if req.Operations != nil { - m.Operations = *req.Operations - } - return m -} - -func applyProviderRequestToModel(dest *model.Provider, req *providerserver.Provider) { - dest.Name = req.Name - dest.ServiceType = req.ServiceType - dest.SchemaVersion = req.SchemaVersion - dest.Endpoint = req.Endpoint - dest.DisplayName = req.DisplayName - dest.Metadata = nil - if req.Metadata != nil { - if metaMap, err := providerMetadataToMap(req.Metadata); err == nil { - dest.Metadata = metaMap - } - } - dest.Operations = nil - if req.Operations != nil { - dest.Operations = *req.Operations - } -} - -func providerMetadataToMap(meta *providerserver.ProviderMetadata) (map[string]interface{}, error) { - b, err := json.Marshal(meta) - if err != nil { - return nil, err - } - var m map[string]interface{} - if err := json.Unmarshal(b, &m); err != nil { - return nil, err - } - return m, nil -} diff --git a/internal/sp/service/provider/provider.go b/internal/sp/service/provider/provider.go deleted file mode 100644 index 18a9732..0000000 --- a/internal/sp/service/provider/provider.go +++ /dev/null @@ -1,316 +0,0 @@ -// Package provider implements business logic for provider management. -package provider - -import ( - "context" - "encoding/base64" - "errors" - "fmt" - "strconv" - "time" - - providerserver "github.com/dcm-project/control-plane/internal/sp/api/provider" - "github.com/dcm-project/control-plane/internal/sp/logging" - "github.com/dcm-project/control-plane/internal/sp/service" - "github.com/dcm-project/control-plane/internal/sp/store" - "github.com/dcm-project/control-plane/internal/sp/store/model" - providerstore "github.com/dcm-project/control-plane/internal/sp/store/provider" - "github.com/google/uuid" -) - -const ( - defaultPageSize = 100 - maxPageSize = 100 -) - -// ListResult contains the result of listing providers with pagination info. -type ListResult struct { - Providers []providerserver.Provider - NextPageToken string -} - -// ProviderService handles business logic for provider management. -type ProviderService struct { - store store.Store -} - -// NewProviderService creates a new ProviderService with the given store. -func NewProviderService(store store.Store) *ProviderService { - return &ProviderService{store: store} -} - -// RegisterOrUpdateProvider implements idempotent provider registration per the DCM spec. -// Returns the provider and a boolean indicating whether an existing provider was updated (true) or a new one was created (false). -// Returns service.ErrCodeConflict if name exists with different ID or ID exists with different name. -func (s *ProviderService) RegisterOrUpdateProvider(ctx context.Context, req *providerserver.Provider, queryID *string) (*providerserver.Provider, bool, error) { - log := logging.FromContext(ctx) - log.Debug("RegisterOrUpdateProvider request received", "name", req.Name, "client_id", queryID) - - requestedID := s.parseProviderID(req.Id, queryID) - - existing, err := s.findExistingByName(ctx, req.Name, requestedID) - if err != nil { - return nil, false, err - } - - if existing != nil { - log.Debug("Existing provider found, updating", "provider_id", existing.ID, "name", req.Name) - existing.ConsecutiveFailures = 0 - existing.NextHealthCheck = nil - updated, err := s.updateExistingProvider(ctx, existing, req) - if err != nil { - return nil, false, err - } - return ModelToProvider(updated), true, nil - } - - providerID, err := s.resolveProviderID(ctx, requestedID) - if err != nil { - return nil, false, err - } - - providerModel := ProviderToModel(req, *providerID) - created, err := s.store.Provider().Create(ctx, providerModel) - if err != nil { - log.Error("Failed to create provider in store", "name", req.Name, "error", err) - return nil, false, err - } - - log.Info("Provider created", "provider_id", created.ID, "name", created.Name) - return ModelToProvider(created), false, nil -} - -// parseProviderID extracts the provider ID from request body or query parameter. -func (s *ProviderService) parseProviderID(bodyID *string, queryID *string) *string { - if bodyID != nil { - id := bodyID - return id - } - if queryID != nil { - id := queryID - return id - } - return nil -} - -// findExistingByName returns the existing provider if name exists and is valid for update. -// Returns service.ErrCodeConflict if name exists with a different ID than requested. -func (s *ProviderService) findExistingByName(ctx context.Context, name string, requestedID *string) (*model.Provider, error) { - log := logging.FromContext(ctx) - - existing, err := s.store.Provider().GetByName(ctx, name) - if err != nil { - if errors.Is(err, providerstore.ErrProviderNotFound) { - return nil, nil - } - log.Error("Failed to look up provider by name", "name", name, "error", err) - return nil, err - } - - if requestedID != nil && existing.ID != *requestedID { - log.Warn("Name conflict detected", "name", name, "existing_id", existing.ID, "requested_id", *requestedID) - return nil, &service.ServiceError{ - Code: service.ErrCodeConflict, - Message: fmt.Sprintf("name '%s' already exists with a different provider ID", name), - } - } - - return existing, nil -} - -// resolveProviderID returns the requested ID after checking for conflicts, or generates a new one. -func (s *ProviderService) resolveProviderID(ctx context.Context, requestedID *string) (*string, error) { - log := logging.FromContext(ctx) - - if requestedID == nil { - generatedID := uuid.New().String() - log.Debug("Generated provider ID", "provider_id", generatedID) - return &generatedID, nil - } - - exists, err := s.store.Provider().ExistsByID(ctx, *requestedID) - if err != nil { - log.Error("Failed to check provider ID existence", "provider_id", *requestedID, "error", err) - return nil, err - } - if exists { - log.Warn("Duplicate provider ID", "provider_id", *requestedID) - return nil, &service.ServiceError{ - Code: service.ErrCodeConflict, - Message: fmt.Sprintf("provider with ID '%s' already exists", *requestedID), - } - } - - return requestedID, nil -} - -func (s *ProviderService) updateExistingProvider(ctx context.Context, existing *model.Provider, req *providerserver.Provider) (*model.Provider, error) { - log := logging.FromContext(ctx) - - applyProviderRequestToModel(existing, req) - existing.UpdateTime = time.Now() - - updated, err := s.store.Provider().Update(ctx, *existing) - if err != nil { - log.Error("Failed to update provider in store", "provider_id", existing.ID, "error", err) - return nil, err - } - - log.Info("Provider updated", "provider_id", updated.ID, "name", updated.Name) - return updated, nil -} - -// GetProvider retrieves a provider by ID. Returns service.ErrCodeNotFound if not found. -func (s *ProviderService) GetProvider(ctx context.Context, providerID string) (*providerserver.Provider, error) { - log := logging.FromContext(ctx) - log.Debug("Getting provider", "provider_id", providerID) - - provider, err := s.store.Provider().Get(ctx, providerID) - if err != nil { - if errors.Is(err, providerstore.ErrProviderNotFound) { - return nil, &service.ServiceError{Code: service.ErrCodeNotFound, Message: fmt.Sprintf("provider %s not found", providerID)} - } - log.Error("Failed to get provider from store", "provider_id", providerID, "error", err) - return nil, err - } - - return ModelToProvider(provider), nil -} - -// ListProviders returns providers with pagination support per AEP-158. -func (s *ProviderService) ListProviders(ctx context.Context, serviceType string, requestedPageSize int, pageToken string) (*ListResult, error) { - log := logging.FromContext(ctx) - log.Debug("Listing providers", - "service_type", serviceType, - "page_size", requestedPageSize, - ) - // Validate and normalize page size per AEP-158 - pageSize := requestedPageSize - if pageSize < 0 { - return nil, &service.ServiceError{Code: service.ErrCodeValidation, Message: "max_page_size must not be negative"} - } - if pageSize == 0 { - pageSize = defaultPageSize - } - if pageSize > maxPageSize { - pageSize = maxPageSize - } - - // Decode page token to get offset - offset := 0 - if pageToken != "" { - decoded, err := DecodePageToken(pageToken) - if err != nil { - return nil, &service.ServiceError{Code: service.ErrCodeValidation, Message: "invalid page_token"} - } - offset = decoded - } - - // Build filter - var filter *providerstore.ProviderFilter - if serviceType != "" { - filter = &providerstore.ProviderFilter{ServiceType: &serviceType} - } - - // Get total count for next page calculation - total, err := s.store.Provider().Count(ctx, filter) - if err != nil { - log.Error("Failed to count providers", "error", err) - return nil, err - } - - // Fetch providers with pagination - pagination := &providerstore.Pagination{Limit: pageSize, Offset: offset} - providers, err := s.store.Provider().List(ctx, filter, pagination) - if err != nil { - log.Error("Failed to list providers from store", "error", err) - return nil, err - } - - // Convert to API types - result := make([]providerserver.Provider, len(providers)) - for i, p := range providers { - result[i] = *ModelToProvider(&p) - } - - // Calculate next page token - var nextPageToken string - nextOffset := offset + len(providers) - if int64(nextOffset) < total { - nextPageToken = encodePageToken(nextOffset) - } - - log.Debug("Providers listed", - "count", len(result), - "has_next_page", nextPageToken != "", - ) - return &ListResult{ - Providers: result, - NextPageToken: nextPageToken, - }, nil -} - -func encodePageToken(offset int) string { - return base64.StdEncoding.EncodeToString([]byte(strconv.Itoa(offset))) -} - -func DecodePageToken(token string) (int, error) { - decoded, err := base64.StdEncoding.DecodeString(token) - if err != nil { - return 0, err - } - return strconv.Atoi(string(decoded)) -} - -// UpdateProvider updates an existing provider. Returns service.ErrCodeNotFound if provider -// doesn't exist, or service.ErrCodeConflict if the new name is already taken. -func (s *ProviderService) UpdateProvider(ctx context.Context, providerID string, update *providerserver.Provider) (*providerserver.Provider, error) { - log := logging.FromContext(ctx) - log.Debug("Updating provider", "provider_id", providerID) - - existing, err := s.store.Provider().Get(ctx, providerID) - if err != nil { - if errors.Is(err, providerstore.ErrProviderNotFound) { - return nil, &service.ServiceError{Code: service.ErrCodeNotFound, Message: fmt.Sprintf("provider %s not found", providerID)} - } - log.Error("Failed to get provider for update", "provider_id", providerID, "error", err) - return nil, err - } - - // Check for name conflict - if update.Name != existing.Name { - other, err := s.store.Provider().GetByName(ctx, update.Name) - if err != nil && !errors.Is(err, providerstore.ErrProviderNotFound) { - log.Error("Failed to check name conflict", "provider_id", providerID, "name", update.Name, "error", err) - return nil, err - } - if other != nil && other.ID != providerID { - log.Warn("Name conflict during update", "provider_id", providerID, "name", update.Name) - return nil, &service.ServiceError{Code: service.ErrCodeConflict, Message: fmt.Sprintf("name '%s' is already taken", update.Name)} - } - } - - updated, err := s.updateExistingProvider(ctx, existing, update) - if err != nil { - return nil, err - } - - return ModelToProvider(updated), nil -} - -// DeleteProvider removes a provider by ID. Returns service.ErrCodeNotFound if not found. -func (s *ProviderService) DeleteProvider(ctx context.Context, providerID string) error { - log := logging.FromContext(ctx) - log.Debug("Deleting provider", "provider_id", providerID) - - err := s.store.Provider().Delete(ctx, providerID) - if err != nil { - if errors.Is(err, providerstore.ErrProviderNotFound) { - return &service.ServiceError{Code: service.ErrCodeNotFound, Message: fmt.Sprintf("provider %s not found", providerID)} - } - log.Error("Failed to delete provider from store", "provider_id", providerID, "error", err) - return err - } - log.Info("Provider deleted", "provider_id", providerID) - return nil -} diff --git a/internal/sp/service/provider/provider_test.go b/internal/sp/service/provider/provider_test.go deleted file mode 100644 index 3c8eeda..0000000 --- a/internal/sp/service/provider/provider_test.go +++ /dev/null @@ -1,455 +0,0 @@ -package provider_test - -import ( - "context" - "fmt" - "time" - - providerserver "github.com/dcm-project/control-plane/internal/sp/api/provider" - "github.com/dcm-project/control-plane/internal/sp/service" - providersvc "github.com/dcm-project/control-plane/internal/sp/service/provider" - "github.com/dcm-project/control-plane/internal/sp/store" - "github.com/dcm-project/control-plane/internal/sp/store/model" - "github.com/google/uuid" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "gorm.io/driver/sqlite" - "gorm.io/gorm" - "gorm.io/gorm/logger" -) - -var _ = Describe("ProviderService", func() { - var ( - db *gorm.DB - dataStore store.Store - providerService *providersvc.ProviderService - ctx context.Context - ) - - BeforeEach(func() { - var err error - db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ - Logger: logger.Default.LogMode(logger.Silent), - }) - Expect(err).NotTo(HaveOccurred()) - Expect(db.AutoMigrate(&model.Provider{})).To(Succeed()) - - dataStore = store.NewStore(db) - providerService = providersvc.NewProviderService(dataStore) - ctx = context.Background() - }) - - AfterEach(func() { - _ = dataStore.Close() - }) - - Describe("RegisterOrUpdateProvider", func() { - It("creates a new provider", func() { - req := newProvider("new-provider") - - resp, updated, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(updated).To(BeFalse()) - Expect(resp.Name).To(Equal("new-provider")) - }) - - It("updates existing provider with same name and ID", func() { - req := newProvider("update-test") - resp1, _, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - - // Re-register with same ID - req.Id = resp1.Id - req.Endpoint = "https://updated.example.com" - _, updated, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(updated).To(BeTrue()) - }) - - It("updates existing provider with same name and no ID (idempotent)", func() { - req := newProvider("idempotent-test") - resp1, _, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - - // Re-register with same name but NO ID - req2 := newProvider("idempotent-test") - req2.Endpoint = "https://updated.example.com" - resp2, updated, err := providerService.RegisterOrUpdateProvider(ctx, req2, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(updated).To(BeTrue()) - Expect(*resp2.Id).To(Equal(*resp1.Id)) // Same ID returned - Expect(resp2.Endpoint).To(Equal("https://updated.example.com")) - }) - - It("persists display_name on create and get", func() { - dn := "Human-readable name" - req := newProvider("persist-display-name") - req.DisplayName = &dn - - resp, _, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.DisplayName).NotTo(BeNil()) - Expect(*resp.DisplayName).To(Equal(dn)) - - got, err := providerService.GetProvider(ctx, *resp.Id) - Expect(err).NotTo(HaveOccurred()) - Expect(got.DisplayName).NotTo(BeNil()) - Expect(*got.DisplayName).To(Equal(dn)) - }) - - It("persists operations on create and get", func() { - ops := []string{"CREATE", "DELETE", "READ"} - req := newProvider("persist-operations") - req.Operations = &ops - - resp, _, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Operations).NotTo(BeNil()) - Expect(*resp.Operations).To(Equal(ops)) - - got, err := providerService.GetProvider(ctx, *resp.Id) - Expect(err).NotTo(HaveOccurred()) - Expect(got.Operations).NotTo(BeNil()) - Expect(*got.Operations).To(Equal(ops)) - }) - - It("persists metadata on create and get", func() { - region := "us-east-1" - req := newProvider("persist-metadata") - req.Metadata = &providerserver.ProviderMetadata{RegionCode: ®ion} - - resp, _, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.Metadata).NotTo(BeNil()) - Expect(resp.Metadata.RegionCode).NotTo(BeNil()) - Expect(*resp.Metadata.RegionCode).To(Equal(region)) - - got, err := providerService.GetProvider(ctx, *resp.Id) - Expect(err).NotTo(HaveOccurred()) - Expect(got.Metadata).NotTo(BeNil()) - Expect(*got.Metadata.RegionCode).To(Equal(region)) - }) - - It("updates metadata on re-register", func() { - req := newProvider("meta-re-register") - req.Metadata = &providerserver.ProviderMetadata{} - req.Metadata.Set("supportedPlatforms", "baremetal") - - resp1, _, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - val, ok := resp1.Metadata.Get("supportedPlatforms") - Expect(ok).To(BeTrue()) - Expect(val).To(Equal("baremetal")) - - req2 := newProvider("meta-re-register") - req2.Id = resp1.Id - req2.Metadata = &providerserver.ProviderMetadata{} - req2.Metadata.Set("supportedPlatforms", "kubevirt") - - resp2, updated, err := providerService.RegisterOrUpdateProvider(ctx, req2, nil) - Expect(err).NotTo(HaveOccurred()) - Expect(updated).To(BeTrue()) - val2, ok := resp2.Metadata.Get("supportedPlatforms") - Expect(ok).To(BeTrue()) - Expect(val2).To(Equal("kubevirt")) - - got, err := providerService.GetProvider(ctx, *resp1.Id) - Expect(err).NotTo(HaveOccurred()) - val3, ok := got.Metadata.Get("supportedPlatforms") - Expect(ok).To(BeTrue()) - Expect(val3).To(Equal("kubevirt")) - }) - - It("resets health backoff on re-register", func() { - req := newProvider("health-reset") - resp, _, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - - futureCheck := time.Now().Add(24 * time.Hour) - err = db.Model(&model.Provider{}).Where("id = ?", *resp.Id).Updates(map[string]any{ - "health_status": model.HealthStatusUnavailable, - "consecutive_failures": 5, - "next_health_check": futureCheck, - }).Error - Expect(err).NotTo(HaveOccurred()) - - req2 := newProvider("health-reset") - req2.Id = resp.Id - _, updated, err := providerService.RegisterOrUpdateProvider(ctx, req2, nil) - Expect(err).NotTo(HaveOccurred()) - Expect(updated).To(BeTrue()) - - var p model.Provider - Expect(db.Where("id = ?", *resp.Id).First(&p).Error).NotTo(HaveOccurred()) - Expect(p.ConsecutiveFailures).To(Equal(0)) - Expect(p.NextHealthCheck).To(BeNil()) - Expect(p.HealthStatus).To(Equal(model.HealthStatusUnavailable)) - }) - - It("returns conflict when name exists with different ID", func() { - req := newProvider("conflict-name") - _, _, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - - // Try with different ID - newID := uuid.New().String() - req.Id = &newID - _, _, err = providerService.RegisterOrUpdateProvider(ctx, req, nil) - - Expect(err).To(HaveOccurred()) - svcErr, ok := err.(*service.ServiceError) - Expect(ok).To(BeTrue()) - Expect(svcErr.Code).To(Equal(service.ErrCodeConflict)) - }) - - It("returns conflict when providerID exists with different name", func() { - req := newProvider("first-name") - resp, _, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - - // Try with same ID but different name - req2 := newProvider("second-name") - req2.Id = resp.Id - _, _, err = providerService.RegisterOrUpdateProvider(ctx, req2, nil) - - Expect(err).To(HaveOccurred()) - svcErr, ok := err.(*service.ServiceError) - Expect(ok).To(BeTrue()) - Expect(svcErr.Code).To(Equal(service.ErrCodeConflict)) - }) - }) - - Describe("GetProvider", func() { - It("returns the provider", func() { - req := newProvider("get-test") - resp, _, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - - provider, err := providerService.GetProvider(ctx, *resp.Id) - - Expect(err).NotTo(HaveOccurred()) - Expect(provider.Name).To(Equal("get-test")) - }) - - It("returns error for non-existent provider", func() { - _, err := providerService.GetProvider(ctx, uuid.New().String()) - - Expect(err).To(HaveOccurred()) - svcErr, ok := err.(*service.ServiceError) - Expect(ok).To(BeTrue()) - Expect(svcErr.Code).To(Equal(service.ErrCodeNotFound)) - }) - }) - - Describe("ListProviders", func() { - It("returns all providers", func() { - _, _, err := providerService.RegisterOrUpdateProvider(ctx, newProvider("p1"), nil) - Expect(err).NotTo(HaveOccurred()) - _, _, err = providerService.RegisterOrUpdateProvider(ctx, newProvider("p2"), nil) - Expect(err).NotTo(HaveOccurred()) - - result, err := providerService.ListProviders(ctx, "", 0, "") - - Expect(err).NotTo(HaveOccurred()) - Expect(result.Providers).To(HaveLen(2)) - }) - - It("filters by service type", func() { - req1 := newProvider("vm-provider") - req1.ServiceType = "vm" - _, _, err := providerService.RegisterOrUpdateProvider(ctx, req1, nil) - Expect(err).NotTo(HaveOccurred()) - - req2 := newProvider("container-provider") - req2.ServiceType = "container" - _, _, err = providerService.RegisterOrUpdateProvider(ctx, req2, nil) - Expect(err).NotTo(HaveOccurred()) - - result, err := providerService.ListProviders(ctx, "vm", 0, "") - - Expect(err).NotTo(HaveOccurred()) - Expect(result.Providers).To(HaveLen(1)) - }) - - It("returns error for negative page size", func() { - _, err := providerService.ListProviders(ctx, "", -1, "") - - Expect(err).To(HaveOccurred()) - svcErr, ok := err.(*service.ServiceError) - Expect(ok).To(BeTrue()) - Expect(svcErr.Code).To(Equal(service.ErrCodeValidation)) - }) - - It("coerces page size to max", func() { - var err error - for i := 0; i < 5; i++ { - _, _, err = providerService.RegisterOrUpdateProvider(ctx, newProvider(fmt.Sprintf("coerce-p%d", i)), nil) - Expect(err).NotTo(HaveOccurred()) - } - - result, err := providerService.ListProviders(ctx, "", 2, "") - - Expect(err).NotTo(HaveOccurred()) - Expect(result.Providers).To(HaveLen(2)) - Expect(result.NextPageToken).NotTo(BeEmpty()) - }) - - It("paginates through results", func() { - var err error - for i := 0; i < 5; i++ { - _, _, err = providerService.RegisterOrUpdateProvider(ctx, newProvider(fmt.Sprintf("paginate-p%d", i)), nil) - Expect(err).NotTo(HaveOccurred()) - } - - // First page - result1, err := providerService.ListProviders(ctx, "", 2, "") - Expect(err).NotTo(HaveOccurred()) - Expect(result1.Providers).To(HaveLen(2)) - Expect(result1.NextPageToken).NotTo(BeEmpty()) - - // Second page - result2, err := providerService.ListProviders(ctx, "", 2, result1.NextPageToken) - Expect(err).NotTo(HaveOccurred()) - Expect(result2.Providers).To(HaveLen(2)) - Expect(result2.NextPageToken).NotTo(BeEmpty()) - - // Third page (last) - result3, err := providerService.ListProviders(ctx, "", 2, result2.NextPageToken) - Expect(err).NotTo(HaveOccurred()) - Expect(result3.Providers).To(HaveLen(1)) - Expect(result3.NextPageToken).To(BeEmpty()) - }) - - It("returns error for invalid page token", func() { - _, err := providerService.ListProviders(ctx, "", 0, "invalid-token") - - Expect(err).To(HaveOccurred()) - svcErr, ok := err.(*service.ServiceError) - Expect(ok).To(BeTrue()) - Expect(svcErr.Code).To(Equal(service.ErrCodeValidation)) - }) - }) - - Describe("UpdateProvider", func() { - It("updates the provider", func() { - req := newProvider("update-provider") - resp, _, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - - update := &providerserver.Provider{ - Id: resp.Id, - Name: "update-provider", - Endpoint: "https://updated.example.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - } - - updated, err := providerService.UpdateProvider(ctx, *resp.Id, update) - - Expect(err).NotTo(HaveOccurred()) - Expect(updated.Endpoint).To(Equal("https://updated.example.com")) - }) - - It("preserves health backoff on field update", func() { - req := newProvider("health-preserve") - resp, _, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - - futureCheck := time.Now().Add(24 * time.Hour) - err = db.Model(&model.Provider{}).Where("id = ?", *resp.Id).Updates(map[string]any{ - "health_status": model.HealthStatusUnavailable, - "consecutive_failures": 5, - "next_health_check": futureCheck, - }).Error - Expect(err).NotTo(HaveOccurred()) - - update := &providerserver.Provider{ - Name: "health-preserve", - Endpoint: "https://updated.example.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - } - _, err = providerService.UpdateProvider(ctx, *resp.Id, update) - Expect(err).NotTo(HaveOccurred()) - - var p model.Provider - Expect(db.Where("id = ?", *resp.Id).First(&p).Error).NotTo(HaveOccurred()) - Expect(p.ConsecutiveFailures).To(Equal(5)) - Expect(p.NextHealthCheck).NotTo(BeNil()) - Expect(p.HealthStatus).To(Equal(model.HealthStatusUnavailable)) - }) - - It("returns conflict when renaming to existing name", func() { - // Create two providers - _, _, err := providerService.RegisterOrUpdateProvider(ctx, newProvider("original-name"), nil) - Expect(err).NotTo(HaveOccurred()) - var resp2 *providerserver.Provider - resp2, _, err = providerService.RegisterOrUpdateProvider(ctx, newProvider("to-rename"), nil) - Expect(err).NotTo(HaveOccurred()) - - // Try to rename second provider to first provider's name - update := &providerserver.Provider{ - Name: "original-name", - Endpoint: "https://example.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - } - - _, err = providerService.UpdateProvider(ctx, *resp2.Id, update) - - Expect(err).To(HaveOccurred()) - svcErr, ok := err.(*service.ServiceError) - Expect(ok).To(BeTrue()) - Expect(svcErr.Code).To(Equal(service.ErrCodeConflict)) - }) - - It("returns error for non-existent provider", func() { - update := &providerserver.Provider{ - Name: "test", - Endpoint: "https://example.com", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - } - - _, err := providerService.UpdateProvider(ctx, uuid.New().String(), update) - - Expect(err).To(HaveOccurred()) - svcErr, ok := err.(*service.ServiceError) - Expect(ok).To(BeTrue()) - Expect(svcErr.Code).To(Equal(service.ErrCodeNotFound)) - }) - }) - - Describe("DeleteProvider", func() { - It("deletes the provider", func() { - req := newProvider("to-delete") - resp, _, err := providerService.RegisterOrUpdateProvider(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - - err = providerService.DeleteProvider(ctx, *resp.Id) - - Expect(err).NotTo(HaveOccurred()) - }) - - It("returns error for non-existent provider", func() { - err := providerService.DeleteProvider(ctx, uuid.New().String()) - - Expect(err).To(HaveOccurred()) - svcErr, ok := err.(*service.ServiceError) - Expect(ok).To(BeTrue()) - Expect(svcErr.Code).To(Equal(service.ErrCodeNotFound)) - }) - }) -}) - -func newProvider(name string) *providerserver.Provider { - return &providerserver.Provider{ - Name: name, - Endpoint: "https://example.com/api", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - } -} diff --git a/internal/sp/service/resource_manager/convert.go b/internal/sp/service/resource_manager/convert.go index ce7a5bf..6994998 100644 --- a/internal/sp/service/resource_manager/convert.go +++ b/internal/sp/service/resource_manager/convert.go @@ -8,25 +8,19 @@ import ( "github.com/dcm-project/control-plane/internal/sp/store/model" ) -// ProviderResponse represents the response from a provider during instance creation. -type ProviderResponse struct { - ID string `json:"id"` - Status string `json:"status"` -} - // ModelToAPI converts a database model to an API response type. func ModelToAPI(instance *model.ServiceTypeInstance) *resource_manager.ServiceTypeInstance { id := instance.ID path := fmt.Sprintf("service-type-instances/%s", id) result := &resource_manager.ServiceTypeInstance{ - Id: &id, - Path: &path, - ProviderName: instance.ProviderName, - Status: &instance.Status, - Spec: instance.Spec, - CreateTime: service.PtrTime(instance.CreateTime), - UpdateTime: service.PtrTime(instance.UpdateTime), + Id: &id, + Path: &path, + AgentName: instance.AgentName, + Status: &instance.Status, + Spec: instance.Spec, + CreateTime: service.PtrTime(instance.CreateTime), + UpdateTime: service.PtrTime(instance.UpdateTime), } if instance.DeletionStatus != nil { diff --git a/internal/sp/service/resource_manager/service_type_instance.go b/internal/sp/service/resource_manager/service_type_instance.go index 27b007c..b417d80 100644 --- a/internal/sp/service/resource_manager/service_type_instance.go +++ b/internal/sp/service/resource_manager/service_type_instance.go @@ -1,4 +1,4 @@ -// Package resource_manager implements business logic for service type instance management. +// Package resource_manager implements service type instance management. package resource_manager import ( @@ -9,110 +9,193 @@ import ( "time" "github.com/dcm-project/control-plane/api/sp/v1alpha1/resource_manager" + agentstore "github.com/dcm-project/control-plane/internal/agent/store/agent" + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" "github.com/dcm-project/control-plane/internal/sp/logging" + "github.com/dcm-project/control-plane/internal/sp/messaging" "github.com/dcm-project/control-plane/internal/sp/service" "github.com/dcm-project/control-plane/internal/sp/store" "github.com/dcm-project/control-plane/internal/sp/store/model" - providerstore "github.com/dcm-project/control-plane/internal/sp/store/provider" rmstore "github.com/dcm-project/control-plane/internal/sp/store/resource_manager" - "github.com/go-resty/resty/v2" "github.com/google/uuid" ) type InstanceService struct { store store.Store - httpClient *resty.Client + publisher *messaging.Publisher + agentStore agentstore.Agent } -func defaultProviderHTTPClient() *resty.Client { - return resty.New(). - SetTimeout(30 * time.Second). - SetRetryCount(3). - SetRetryWaitTime(2 * time.Second). - SetRetryMaxWaitTime(30 * time.Second) -} - -// NewInstanceService constructs InstanceService. If httpClient is nil, production -// defaults are used for outbound provider HTTP; tests may pass a non-nil client -// (e.g. no retries, short timeout) to avoid slow failures. -func NewInstanceService(store store.Store, httpClient *resty.Client) *InstanceService { - if httpClient == nil { - httpClient = defaultProviderHTTPClient() - } +// NewInstanceService constructs InstanceService. publisher may be nil (e.g. in +// tests); when nil, NATS publishing is skipped. +func NewInstanceService(store store.Store, publisher *messaging.Publisher, agentSt agentstore.Agent) *InstanceService { return &InstanceService{ store: store, - httpClient: httpClient, + publisher: publisher, + agentStore: agentSt, } } -// CreateInstance creates a new service type instance -func (s *InstanceService) CreateInstance(ctx context.Context, request *resource_manager.ServiceTypeInstance, queryID *string) (*resource_manager.ServiceTypeInstance, error) { +func (s *InstanceService) CreateInstance(ctx context.Context, request *resource_manager.ServiceTypeInstance, queryID *string, agentName string) (*resource_manager.ServiceTypeInstance, error) { log := logging.FromContext(ctx) - providerName := request.ProviderName - log.Debug("Creating instance", "provider_name", providerName) + serviceType, ok := request.Spec["service_type"].(string) + if !ok { + return nil, service.NewValidationError("spec.service_type is required and must be a string") + } + if strings.TrimSpace(serviceType) == "" { + return nil, service.NewValidationError("spec.service_type must not be empty") + } - provider, err := s.store.Provider().GetByName(ctx, providerName) - if err != nil { - if errors.Is(err, providerstore.ErrProviderNotFound) { - return nil, service.NewNotFoundError(fmt.Sprintf("provider '%s' not found", providerName)) - } - log.Error("Failed to retrieve provider", "provider_name", providerName, "error", err) - return nil, service.NewInternalError(fmt.Sprintf("failed to retrieve provider: %v", err)) + // An empty agentName would create a permanently orphaned Pending row: + // sweepPending's query filters on `agent_name IS NOT NULL`. + if strings.TrimSpace(agentName) == "" { + return nil, service.NewValidationError("agent_name is required and must not be empty") } - // Check Provider if provider is not in ready state - if provider.HealthStatus != model.HealthStatusReady { - log.Warn("Provider not in ready state", "provider_name", providerName, "health_status", provider.HealthStatus) - return nil, service.NewProviderError(fmt.Sprintf("provider '%s' is not in ready state (current status: %s)", providerName, provider.HealthStatus)) + if s.publisher == nil { + return nil, service.NewUnavailableError("nats publisher unavailable, cannot dispatch to agent") + } + if err := s.validateAgent(ctx, agentName, serviceType); err != nil { + return nil, err } - // Resolve instance ID instanceID, err := s.resolveInstanceID(ctx, queryID) if err != nil { return nil, err } - // Extract service_type from spec — ok==false if the key is absent or the value is not a string - serviceType, ok := request.Spec["service_type"].(string) - if !ok { - return nil, service.NewValidationError("spec.service_type is required and must be a string") + now := time.Now() + instance := model.ServiceTypeInstance{ + ID: *instanceID, + ServiceType: serviceType, + Status: model.StatusPending, + Spec: request.Spec, + AgentName: &agentName, + PendingStartedAt: &now, } - if strings.TrimSpace(serviceType) == "" { - return nil, service.NewValidationError("spec.service_type must not be empty") + + created, err := s.store.ServiceTypeInstance().Create(ctx, instance) + if err != nil { + log.Error("Failed to create instance in store", "instance_id", *instanceID, "error", err) + return nil, service.NewInternalError(fmt.Sprintf("failed to create database record for instance %s: %v", *instanceID, err)) + } + + subject, pubErr := s.resolveAgentSubject(ctx, agentName) + if pubErr != nil { + log.Error("Failed to resolve agent topic, rolling back instance", "agent_name", agentName, "error", pubErr) + _ = s.store.ServiceTypeInstance().HardDelete(ctx, created.ID) + return nil, service.NewProvisioningError(fmt.Sprintf("agent '%s' topic resolution failed: %v", agentName, pubErr)) + } + + pubErr = s.publisher.PublishCreate(ctx, subject, messaging.CreatePayload{ + ResourceID: created.ID, + ServiceType: serviceType, + Spec: request.Spec, + }) + if pubErr != nil { + log.Error("Failed to publish create event, rolling back instance", "instance_id", created.ID, "error", pubErr) + _ = s.store.ServiceTypeInstance().HardDelete(ctx, created.ID) + return nil, service.NewProvisioningError(fmt.Sprintf("failed to publish create request for instance %s: %v", created.ID, pubErr)) } - // Send request to provider endpoint with the resolved ID - providerResponse, err := s.createInstanceWithProvider(ctx, provider.Endpoint, request, instanceID) + log.Info("Instance created", "instance_id", created.ID, "status", created.Status, "agent_name", agentName) + return ModelToAPI(created), nil +} + +// ReassignAgent re-points an existing instance at a new agent and re-triggers +// provisioning from scratch (fresh "pending" state, retry count reset). Used +// by the self-healing loop when the originally assigned agent fails, times +// out, or is excluded from re-evaluation. +// +// expectedCurrentAgent is CASed against agent_name in the same update as the +// status check, and must be the caller's own observation of the instance's +// agent (e.g. the excluded agent), not derived from a fresh read here: a +// fresh read would just reflect whatever the most recent writer set, +// silently turning the CAS into an unconditional overwrite and defeating the +// cross-replica/sibling-heal race it exists to catch. +func (s *InstanceService) ReassignAgent(ctx context.Context, instanceID string, agentName string, expectedCurrentAgent string) error { + log := logging.FromContext(ctx) + + instance, err := s.store.ServiceTypeInstance().Get(ctx, instanceID, false) if err != nil { - log.Error("Provider provisioning failed", "instance_id", *instanceID, "provider_name", providerName, "error", err) - return nil, service.NewProviderError(fmt.Sprintf("Error from Provider (%s): %v", providerName, err)) + if errors.Is(err, rmstore.ErrInstanceNotFound) { + return service.NewNotFoundError(fmt.Sprintf("instance %s not found", instanceID)) + } + return service.NewInternalError(fmt.Sprintf("failed to retrieve instance: %v", err)) } - // Create instance in database - instance := model.ServiceTypeInstance{ - ID: *instanceID, - ProviderName: providerName, - ServiceType: serviceType, - Status: providerResponse.Status, - Spec: request.Spec, + if err := s.validateAgent(ctx, agentName, instance.ServiceType); err != nil { + return err } - created, err := s.store.ServiceTypeInstance().Create(ctx, instance) + if s.publisher == nil { + return service.NewUnavailableError("nats publisher unavailable, cannot reassign instance") + } + + subject, err := s.resolveAgentSubject(ctx, agentName) if err != nil { - log.Error("Failed to create instance in store", "instance_id", *instanceID, "error", err) - return nil, service.NewInternalError(fmt.Sprintf("failed to create database record for instance %s: %v", providerResponse.ID, err)) + return service.NewProvisioningError(fmt.Sprintf("agent '%s' topic resolution failed: %v", agentName, err)) } - log.Info("Instance created successfully", - "instance_id", created.ID, - "provider_name", providerName, - "status", providerResponse.Status, - ) - return ModelToAPI(created), nil + if err := s.store.ServiceTypeInstance().ReassignAndReset(ctx, instanceID, agentName, expectedCurrentAgent); err != nil { + if errors.Is(err, rmstore.ErrInstanceNotFound) { + return service.NewNotFoundError(fmt.Sprintf("instance %s not found", instanceID)) + } + if errors.Is(err, rmstore.ErrInstanceNotEligible) { + return service.NewConflictError(fmt.Sprintf("instance %s is being deleted and cannot be reassigned", instanceID)) + } + return service.NewInternalError(fmt.Sprintf("failed to reassign instance %s: %v", instanceID, err)) + } + + if pubErr := s.publisher.PublishCreate(ctx, subject, messaging.CreatePayload{ + ResourceID: instanceID, + ServiceType: instance.ServiceType, + Spec: instance.Spec, + }); pubErr != nil { + log.Error("Failed to publish create after reassignment", "instance_id", instanceID, "agent_name", agentName, "error", pubErr) + return service.NewProvisioningError(fmt.Sprintf("failed to publish create for reassigned instance %s: %v", instanceID, pubErr)) + } + + log.Info("Instance reassigned to new agent", "instance_id", instanceID, "agent_name", agentName) + return nil +} + +func (s *InstanceService) resolveAgentSubject(ctx context.Context, agentName string) (string, error) { + agent, err := s.agentStore.GetByName(ctx, agentName) + if err != nil { + return "", err + } + return agent.TopicName, nil +} + +func (s *InstanceService) validateAgent(ctx context.Context, agentName string, serviceType string) error { + if strings.TrimSpace(agentName) == "" { + return service.NewValidationError("agent_name is required and must not be empty") + } + if s.agentStore == nil { + return service.NewUnavailableError("agent store unavailable, cannot validate or dispatch to agent") + } + agent, err := s.agentStore.GetByName(ctx, agentName) + if err != nil { + if errors.Is(err, agentstore.ErrAgentNotFound) { + return service.NewNotFoundError(fmt.Sprintf("agent '%s' not found", agentName)) + } + return service.NewInternalError(fmt.Sprintf("failed to look up agent '%s': %v", agentName, err)) + } + + if agent.HealthStatus != agentmodel.AgentHealthStatusReady { + return service.NewUnavailableError(fmt.Sprintf("agent '%s' is %s", agentName, agent.HealthStatus)) + } + + for _, st := range agent.ServiceTypes { + if st == serviceType { + return nil + } + } + return service.NewValidationError(fmt.Sprintf("agent '%s' does not serve service type '%s'", agentName, serviceType)) } -// GetInstance retrieves an instance by ID func (s *InstanceService) GetInstance(ctx context.Context, instanceID string, showDeleted bool) (*resource_manager.ServiceTypeInstance, error) { log := logging.FromContext(ctx) log.Debug("Getting instance", "instance_id", instanceID, "show_deleted", showDeleted) @@ -129,23 +212,15 @@ func (s *InstanceService) GetInstance(ctx context.Context, instanceID string, sh return ModelToAPI(instance), nil } -// ListInstances returns instances with optional filtering and pagination -func (s *InstanceService) ListInstances(ctx context.Context, providerName *string, serviceType *string, showDeleted bool, maxPageSize *int, pageToken *string) (*resource_manager.ServiceTypeInstanceList, error) { +func (s *InstanceService) ListInstances(ctx context.Context, serviceType, agentName *string, showDeleted bool, maxPageSize *int, pageToken *string) (*resource_manager.ServiceTypeInstanceList, error) { log := logging.FromContext(ctx) - log.Debug("Listing instances", - "provider_filter", providerName, - "service_type_filter", serviceType, - "page_size", maxPageSize, - "show_deleted", showDeleted, - ) opts := &rmstore.ServiceTypeInstanceListOptions{ - ProviderName: providerName, - ServiceType: serviceType, - ShowDeleted: showDeleted, + ServiceType: serviceType, + AgentName: agentName, + ShowDeleted: showDeleted, } - // Apply max page size (default 50, max 100) if maxPageSize != nil { if *maxPageSize > 0 && *maxPageSize <= 100 { opts.PageSize = *maxPageSize @@ -154,7 +229,6 @@ func (s *InstanceService) ListInstances(ctx context.Context, providerName *strin } } - // Apply page token if pageToken != nil && *pageToken != "" { opts.PageToken = pageToken } @@ -165,178 +239,139 @@ func (s *InstanceService) ListInstances(ctx context.Context, providerName *strin return nil, service.NewInternalError(fmt.Sprintf("failed to list instances: %v", err)) } - // Convert to API types apiInstances := make([]resource_manager.ServiceTypeInstance, len(result.Instances)) for i, inst := range result.Instances { apiInstances[i] = *ModelToAPI(&inst) } - log.Debug("Instances listed", - "count", len(apiInstances), - "has_next_page", result.NextPageToken != nil, - ) - apiResult := &resource_manager.ServiceTypeInstanceList{ + return &resource_manager.ServiceTypeInstanceList{ Instances: &apiInstances, NextPageToken: result.NextPageToken, - } - - return apiResult, nil + }, nil } -// DeleteInstance removes an instance by ID. When deferred is true, the instance -// is marked for background cleanup without contacting the provider. +// DeleteInstance removes an instance. Deferred: marks for background cleanup +// and swallows publish failures for the cleanup scheduler to retry. +// Non-deferred: publish failures return an error to the caller immediately. +// In both cases, an agent-routed instance is only purged once the agent's +// "deletion-acknowledged" event confirms the physical resource is gone (see +// consumer.ResponseConsumer). A non-deferred delete is therefore also +// enrolled in the same deletion_status=SCHEDULED retry/audit-giveup +// tracking the cleanup scheduler uses for deferred deletes. func (s *InstanceService) DeleteInstance(ctx context.Context, instanceID string, deferred bool) error { log := logging.FromContext(ctx) - log.Debug("Deleting instance", "instance_id", instanceID, "deferred", deferred) - // Get instance to find provider (include soft-deleted so we can retry cleanup) instance, err := s.store.ServiceTypeInstance().Get(ctx, instanceID, true) if err != nil { if errors.Is(err, rmstore.ErrInstanceNotFound) { return service.NewNotFoundError(fmt.Sprintf("instance %s not found", instanceID)) } - log.Error("Failed to get instance for deletion", "instance_id", instanceID, "error", err) return service.NewInternalError(fmt.Sprintf("failed to retrieve instance: %v", err)) } - // Deferred mode: skip provider call, just enqueue for background cleanup if deferred { if instance.DeletionStatus != nil { - // Already marked — reset retry count so scheduler picks it up again if resetErr := s.store.ServiceTypeInstance().ResetRetryCount(ctx, instanceID); resetErr != nil { - log.Error("Failed to reset retry count for instance", "instance_id", instanceID, "error", resetErr) + log.Error("Failed to reset retry count", "instance_id", instanceID, "error", resetErr) } } else { - // Mark as pending deletion if markErr := s.store.ServiceTypeInstance().MarkForDeletion(ctx, instanceID); markErr != nil { return service.NewInternalError(fmt.Sprintf("failed to mark instance %s for deletion: %v", instanceID, markErr)) } } - log.Info("Scheduled deferred deletion of instance from provider", "instance_id", instance.ID, "provider_name", instance.ProviderName) + if err := s.publishDeleteToAgent(ctx, instance); err != nil { + log.Warn("Failed to publish deferred delete to agent, sweep will retry", + "instance_id", instanceID, "error", err) + } + log.Info("Scheduled deferred deletion", "instance_id", instanceID) return nil } - // Non-deferred: attempt SP deletion and DB hard-delete - deleteErr := s.DeleteFromProvider(ctx, instance) - if deleteErr == nil { + // Nothing to wait on: never agent-routed, or no publisher configured to + // dispatch a delete request in the first place. Delete now, matching the + // cleanup scheduler's own handling of the same case. + if instance.AgentName == nil || s.publisher == nil { + if err := s.store.ServiceTypeInstance().HardDelete(ctx, instanceID); err != nil { + return service.NewInternalError(fmt.Sprintf("failed to delete instance %s: %v", instanceID, err)) + } + log.Info("Instance deleted (no agent to notify)", "instance_id", instance.ID) return nil } - log.Error( - "Failed to delete instance from provider", - "instance_id", instance.ID, - "provider_name", instance.ProviderName, - "error", deleteErr, - ) - - // For already-pending/failed instances, reset retry count so scheduler picks it up again - if instance.DeletionStatus != nil { - if resetErr := s.store.ServiceTypeInstance().ResetRetryCount(ctx, instanceID); resetErr != nil { - log.Error("Failed to reset retry count for instance", "instance_id", instanceID, "error", resetErr) + if err := s.publishDeleteToAgent(ctx, instance); err != nil { + if errors.Is(err, agentstore.ErrAgentNotFound) { + // The agent is gone, so no "deletion-acknowledged" will ever + // arrive: purge now instead of stranding the instance in + // "deleting" forever, matching the cleanup scheduler's own + // audit-giveup behavior for the deferred path. + log.Warn("Agent not found for non-deferred delete, deleting locally without confirmation", + "instance_id", instanceID, "agent_name", *instance.AgentName) + if hardErr := s.store.ServiceTypeInstance().HardDelete(ctx, instanceID); hardErr != nil { + return service.NewInternalError(fmt.Sprintf("failed to delete instance %s: %v", instanceID, hardErr)) + } + return nil } + log.Error("Failed to publish delete to agent", "instance_id", instanceID, "error", err) + return service.NewProvisioningError(fmt.Sprintf("failed to publish delete for instance %s: %v", instanceID, err)) } - return service.NewProviderError(fmt.Sprintf("failed to delete instance (%s): %v", instanceID, deleteErr)) -} -// DeleteFromProvider deletes the instance from its service provider and, on -// success, hard-deletes the database record. -func (s *InstanceService) DeleteFromProvider(ctx context.Context, instance *model.ServiceTypeInstance) error { - log := logging.FromContext(ctx) - log.Debug("Deleting instance from provider", "instance_id", instance.ID, "provider_name", instance.ProviderName) - - provider, err := s.store.Provider().GetByName(ctx, instance.ProviderName) - if err != nil { - if errors.Is(err, providerstore.ErrProviderNotFound) { - return fmt.Errorf("provider '%s' not found", instance.ProviderName) - } - return fmt.Errorf("failed to retrieve provider: %w", err) + if err := s.store.ServiceTypeInstance().UpdateStatus(ctx, instanceID, model.StatusDeleting, ""); err != nil { + log.Error("Failed to mark instance deleting", "instance_id", instanceID, "error", err) + return service.NewInternalError(fmt.Sprintf("failed to update instance %s: %v", instanceID, err)) } - if err = s.deleteInstanceWithProvider(ctx, provider.Endpoint, instance.ID); err != nil { - return err + if err := s.store.ServiceTypeInstance().MarkForDeletion(ctx, instanceID); err != nil { + // Best-effort: the instance is already "deleting" and a prompt ack + // still finalizes it via handleDeletionAcknowledged even without + // retry tracking; it just won't be retried/audited if the ack never + // arrives until the next code path touches it. + log.Error("Failed to enroll non-deferred delete in cleanup retry tracking", "instance_id", instanceID, "error", err) } - log.Info("Instance deleted successfully", - "instance_id", instance.ID, - "provider_name", instance.ProviderName, - ) + log.Info("Delete requested, awaiting agent acknowledgement", "instance_id", instance.ID) + return nil +} - if err = s.store.ServiceTypeInstance().HardDelete(ctx, instance.ID); err != nil { - return fmt.Errorf("failed to delete database record for instance %s: %w", instance.ID, err) +// publishDeleteToAgent publishes a delete request to the instance's agent. +// Callers are responsible for deciding what an agentstore.ErrAgentNotFound +// error means for their delete path: the deferred path treats any error +// (including this one) as "log and let the cleanup scheduler retry", which +// will itself hit the same ErrAgentNotFound on its own lookup and audit-give-up; +// the non-deferred path (DeleteInstance) must react to it directly instead of +// silently treating a missing agent as a successful publish. +func (s *InstanceService) publishDeleteToAgent(ctx context.Context, instance *model.ServiceTypeInstance) error { + if s.publisher == nil || instance.AgentName == nil { + return nil } - - log.Info("Deleted instance from DB record", "instance_id", instance.ID) - return nil + subject, err := s.resolveAgentSubject(ctx, *instance.AgentName) + if err != nil { + return err + } + return s.publisher.PublishDelete(ctx, subject, messaging.DeletePayload{ + ResourceID: instance.ID, + ServiceType: instance.ServiceType, + }) } -// resolveInstanceID returns the requested ID after checking for conflicts, or generates a new one func (s *InstanceService) resolveInstanceID(ctx context.Context, queryID *string) (*string, error) { log := logging.FromContext(ctx) if queryID == nil || *queryID == "" { - generatedId := uuid.New().String() - log.Debug("Generated instance ID", "instance_id", generatedId) - return &generatedId, nil + generatedID := uuid.New().String() + log.Debug("Generated instance ID", "instance_id", generatedID) + return &generatedID, nil } requestedID := *queryID exists, err := s.store.ServiceTypeInstance().ExistsByID(ctx, requestedID) if err != nil { - log.Error("Failed to check instance ID existence", "instance_id", requestedID, "error", err) return nil, service.NewInternalError(fmt.Sprintf("failed to check instance existence: %v", err)) } if exists { - log.Warn("Duplicate instance ID", "instance_id", requestedID) return nil, service.NewConflictError(fmt.Sprintf("instance with ID '%s' already exists", requestedID)) } return &requestedID, nil } - -// createInstanceWithProvider sends the create request to the provider's endpoint -func (s *InstanceService) createInstanceWithProvider(ctx context.Context, endpoint string, request *resource_manager.ServiceTypeInstance, id *string) (*ProviderResponse, error) { - log := logging.FromContext(ctx) - - var providerResp ProviderResponse - - resp, err := s.httpClient.R(). - SetContext(ctx). - SetHeader("Content-Type", "application/json"). - SetQueryParam("id", *id). - SetBody(map[string]interface{}{"spec": request.Spec}). - SetResult(&providerResp). - Post(endpoint) - if err != nil { - log.Error("Failed to connect to provider", "endpoint", endpoint, "error", err) - return nil, service.NewProviderError(fmt.Sprintf("failed to connect to provider: %v", err)) - } - - if resp.IsError() { - log.Error("Provider returned error", "endpoint", endpoint, "status", resp.Status()) - return nil, service.NewProviderError(fmt.Sprintf("provider returned error: %s", resp.Status())) - } - - return &providerResp, nil -} - -// deleteInstanceWithProvider sends the delete request to the provider's endpoint -func (s *InstanceService) deleteInstanceWithProvider(ctx context.Context, endpoint string, instanceID string) error { - log := logging.FromContext(ctx) - - resp, err := s.httpClient.R(). - SetContext(ctx). - Delete(fmt.Sprintf("%s/%s", endpoint, instanceID)) - if err != nil { - log.Error("Failed to connect to provider for deletion", "endpoint", endpoint, "instance_id", instanceID, "error", err) - return fmt.Errorf("failed to connect to provider: %w", err) - } - - if resp.IsError() && resp.StatusCode() != 404 { - log.Error("Provider returned error on deletion", "endpoint", endpoint, "instance_id", instanceID, "status", resp.Status()) - return fmt.Errorf("provider returned error: %s", resp.Status()) - } - - return nil -} diff --git a/internal/sp/service/resource_manager/service_type_instance_test.go b/internal/sp/service/resource_manager/service_type_instance_test.go index a2f4340..c14124f 100644 --- a/internal/sp/service/resource_manager/service_type_instance_test.go +++ b/internal/sp/service/resource_manager/service_type_instance_test.go @@ -2,19 +2,18 @@ package resource_manager_test import ( "context" - "encoding/json" "errors" - "net/http" - "net/http/httptest" - "time" "github.com/dcm-project/control-plane/api/sp/v1alpha1/resource_manager" + agentStoreImpl "github.com/dcm-project/control-plane/internal/agent/store/agent" + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" + "github.com/dcm-project/control-plane/internal/sp/messaging" "github.com/dcm-project/control-plane/internal/sp/service" rmsvc "github.com/dcm-project/control-plane/internal/sp/service/resource_manager" "github.com/dcm-project/control-plane/internal/sp/store" "github.com/dcm-project/control-plane/internal/sp/store/model" - "github.com/go-resty/resty/v2" "github.com/google/uuid" + "github.com/nats-io/nats.go/jetstream" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "gorm.io/driver/sqlite" @@ -22,15 +21,24 @@ import ( "gorm.io/gorm/logger" ) +// stubJetStream acknowledges every publish so tests can exercise the +// agent-routed CreateInstance/ReassignAgent paths without a real NATS server. +type stubJetStream struct { + jetstream.JetStream +} + +func (s *stubJetStream) Publish(_ context.Context, _ string, _ []byte, _ ...jetstream.PublishOpt) (*jetstream.PubAck, error) { + return &jetstream.PubAck{}, nil +} + +func ptrString(s string) *string { return &s } + var _ = Describe("InstanceService", func() { var ( db *gorm.DB dataStore store.Store instanceService *rmsvc.InstanceService ctx context.Context - mockProvider *httptest.Server - providerCalled bool - deleteRequested bool ) BeforeEach(func() { @@ -39,71 +47,42 @@ var _ = Describe("InstanceService", func() { Logger: logger.Default.LogMode(logger.Silent), }) Expect(err).NotTo(HaveOccurred()) - Expect(db.AutoMigrate(&model.Provider{}, &model.ServiceTypeInstance{})).To(Succeed()) - - // Create a mock provider server - providerCalled = false - deleteRequested = false - mockProvider = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodDelete { - deleteRequested = true - w.WriteHeader(http.StatusNoContent) - return - } - providerCalled = true - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _ = json.NewEncoder(w).Encode(map[string]string{ - "id": uuid.New().String(), - "status": "PROVISIONING", - }) - })) - - // Create a provider in the database - provider := model.Provider{ - ID: uuid.New().String(), - Name: "test-provider", - ServiceType: "vm", - Endpoint: mockProvider.URL, - HealthStatus: model.HealthStatusReady, - } - Expect(db.Create(&provider).Error).NotTo(HaveOccurred()) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) + Expect(db.Create(&agentmodel.Agent{ID: uuid.New().String(), Name: "test-agent", TopicName: "dcm.agent.test-agent", HealthStatus: agentmodel.AgentHealthStatusReady, ServiceTypes: []string{"vm", "container"}}).Error).NotTo(HaveOccurred()) dataStore = store.NewStore(db) - instanceService = rmsvc.NewInstanceService(dataStore, resty.New(). - SetTimeout(5*time.Second). - SetRetryCount(0)) + pub := messaging.NewPublisher(&stubJetStream{}) + instanceService = rmsvc.NewInstanceService(dataStore, pub, agentStoreImpl.NewAgent(db)) ctx = context.Background() }) AfterEach(func() { - mockProvider.Close() _ = dataStore.Close() }) - Describe("CreateInstance", func() { - It("creates a new instance", func() { + Describe("CreateInstance (agent-routed provisioning)", func() { + It("creates instance with pending status via agent NATS", func() { req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "memory": "4GB", "service_type": "vm"}, + Spec: map[string]interface{}{"cpu": 2, "memory": "4GB", "service_type": "vm"}, } - result, err := instanceService.CreateInstance(ctx, req, nil) + result, err := instanceService.CreateInstance(ctx, req, nil, "test-agent") Expect(err).NotTo(HaveOccurred()) Expect(result).NotTo(BeNil()) Expect(result.Id).NotTo(BeNil()) - Expect(result.ProviderName).To(Equal("test-provider")) - Expect(providerCalled).To(BeTrue()) + + var stored model.ServiceTypeInstance + Expect(db.First(&stored, "id = ?", *result.Id).Error).NotTo(HaveOccurred()) + Expect(stored.Status).To(Equal("pending")) }) It("sets service_type from spec", func() { req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, + Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, } - result, err := instanceService.CreateInstance(ctx, req, nil) + result, err := instanceService.CreateInstance(ctx, req, nil, "test-agent") Expect(err).NotTo(HaveOccurred()) var dbInstance model.ServiceTypeInstance @@ -114,11 +93,10 @@ var _ = Describe("InstanceService", func() { It("creates instance with specified ID", func() { specifiedID := uuid.New().String() req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, + Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, } - result, err := instanceService.CreateInstance(ctx, req, &specifiedID) + result, err := instanceService.CreateInstance(ctx, req, &specifiedID, "test-agent") Expect(err).NotTo(HaveOccurred()) Expect(*result.Id).To(Equal(specifiedID)) @@ -127,16 +105,13 @@ var _ = Describe("InstanceService", func() { It("returns conflict error for duplicate ID", func() { specifiedID := uuid.New().String() req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, + Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, } - // First creation should succeed - _, err := instanceService.CreateInstance(ctx, req, &specifiedID) + _, err := instanceService.CreateInstance(ctx, req, &specifiedID, "test-agent") Expect(err).NotTo(HaveOccurred()) - // Second creation with same ID should fail - _, err = instanceService.CreateInstance(ctx, req, &specifiedID) + _, err = instanceService.CreateInstance(ctx, req, &specifiedID, "test-agent") Expect(err).To(HaveOccurred()) var svcErr *service.ServiceError @@ -145,176 +120,56 @@ var _ = Describe("InstanceService", func() { Expect(svcErr.Code).To(Equal(service.ErrCodeConflict)) }) - It("returns not found error for non-existent provider", func() { - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "non-existent-provider", - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, - } - - _, err := instanceService.CreateInstance(ctx, req, nil) - - Expect(err).To(HaveOccurred()) - var svcErr *service.ServiceError - Expect(err).To(BeAssignableToTypeOf(svcErr)) - errors.As(err, &svcErr) - Expect(svcErr.Code).To(Equal(service.ErrCodeNotFound)) - }) - - It("returns provider error when provider exists but is not ready", func() { - // Create a provider with HealthStatus = NotReady - notReadyProvider := model.Provider{ - ID: uuid.New().String(), - Name: "not-ready-provider", - ServiceType: "vm", - Endpoint: mockProvider.URL, - HealthStatus: model.HealthStatusUnavailable, - } - Expect(db.Create(¬ReadyProvider).Error).NotTo(HaveOccurred()) - - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "not-ready-provider", - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, - } - - _, err := instanceService.CreateInstance(ctx, req, nil) - - Expect(err).To(HaveOccurred()) - var svcErr *service.ServiceError - Expect(err).To(BeAssignableToTypeOf(svcErr)) - errors.As(err, &svcErr) - Expect(svcErr.Code).To(Equal(service.ErrCodeProviderError)) - Expect(svcErr.Message).To(ContainSubstring("not in ready state")) - }) - - It("returns provider error when provider endpoint fails", func() { - // Create a provider with a bad endpoint - badProvider := model.Provider{ - ID: uuid.New().String(), - Name: "bad-provider", - ServiceType: "vm", - Endpoint: "http://localhost:1", // Invalid port - } - Expect(db.Create(&badProvider).Error).NotTo(HaveOccurred()) - - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "bad-provider", - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, - } - - _, err := instanceService.CreateInstance(ctx, req, nil) - - Expect(err).To(HaveOccurred()) - var svcErr *service.ServiceError - Expect(err).To(BeAssignableToTypeOf(svcErr)) - errors.As(err, &svcErr) - Expect(svcErr.Code).To(Equal(service.ErrCodeProviderError)) - }) - - It("returns provider error when provider responds with 4xx HTTP error", func() { - // Create a mock server that returns 400 - mockProvider4xx := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(`{"error": "bad request"}`)) - })) - defer mockProvider4xx.Close() - - provider4xx := model.Provider{ - ID: uuid.New().String(), - Name: "provider-4xx", - ServiceType: "vm", - Endpoint: mockProvider4xx.URL, - HealthStatus: model.HealthStatusReady, - } - Expect(db.Create(&provider4xx).Error).NotTo(HaveOccurred()) - - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "provider-4xx", - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, - } - - _, err := instanceService.CreateInstance(ctx, req, nil) - - Expect(err).To(HaveOccurred()) - var svcErr *service.ServiceError - Expect(err).To(BeAssignableToTypeOf(svcErr)) - errors.As(err, &svcErr) - Expect(svcErr.Code).To(Equal(service.ErrCodeProviderError)) - Expect(svcErr.Message).To(ContainSubstring("provider returned error")) - }) - - It("returns provider error when provider responds with 5xx HTTP error", func() { - // Create a mock server that returns 500 - mockProvider5xx := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(`{"error": "internal server error"}`)) - })) - defer mockProvider5xx.Close() - - provider5xx := model.Provider{ - ID: uuid.New().String(), - Name: "provider-5xx", - ServiceType: "vm", - Endpoint: mockProvider5xx.URL, - HealthStatus: model.HealthStatusReady, - } - Expect(db.Create(&provider5xx).Error).NotTo(HaveOccurred()) - + It("returns validation error when spec is missing service_type", func() { req := &resource_manager.ServiceTypeInstance{ - ProviderName: "provider-5xx", - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, + Spec: map[string]interface{}{"cpu": 2}, } - _, err := instanceService.CreateInstance(ctx, req, nil) + _, err := instanceService.CreateInstance(ctx, req, nil, "test-agent") Expect(err).To(HaveOccurred()) var svcErr *service.ServiceError Expect(err).To(BeAssignableToTypeOf(svcErr)) errors.As(err, &svcErr) - Expect(svcErr.Code).To(Equal(service.ErrCodeProviderError)) - Expect(svcErr.Message).To(ContainSubstring("provider returned error")) + Expect(svcErr.Code).To(Equal(service.ErrCodeValidation)) + Expect(svcErr.Message).To(ContainSubstring("spec.service_type is required")) }) - It("returns validation error when spec is missing service_type", func() { + It("returns validation error when spec.service_type is not a string", func() { req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2}, + Spec: map[string]interface{}{"cpu": 2, "service_type": 42}, } - _, err := instanceService.CreateInstance(ctx, req, nil) + _, err := instanceService.CreateInstance(ctx, req, nil, "test-agent") Expect(err).To(HaveOccurred()) var svcErr *service.ServiceError Expect(err).To(BeAssignableToTypeOf(svcErr)) errors.As(err, &svcErr) Expect(svcErr.Code).To(Equal(service.ErrCodeValidation)) - Expect(svcErr.Message).To(ContainSubstring("spec.service_type is required")) - Expect(providerCalled).To(BeFalse()) }) - It("returns validation error when spec.service_type is not a string", func() { + It("returns validation error when spec.service_type is empty", func() { req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": 42}, + Spec: map[string]interface{}{"cpu": 2, "service_type": ""}, } - _, err := instanceService.CreateInstance(ctx, req, nil) + _, err := instanceService.CreateInstance(ctx, req, nil, "test-agent") Expect(err).To(HaveOccurred()) var svcErr *service.ServiceError Expect(err).To(BeAssignableToTypeOf(svcErr)) errors.As(err, &svcErr) Expect(svcErr.Code).To(Equal(service.ErrCodeValidation)) - Expect(svcErr.Message).To(ContainSubstring("spec.service_type is required")) - Expect(providerCalled).To(BeFalse()) + Expect(svcErr.Message).To(ContainSubstring("must not be empty")) }) - It("returns validation error when spec.service_type is an empty string", func() { + It("returns validation error when spec.service_type is whitespace only", func() { req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": ""}, + Spec: map[string]interface{}{"cpu": 2, "service_type": " "}, } - _, err := instanceService.CreateInstance(ctx, req, nil) + _, err := instanceService.CreateInstance(ctx, req, nil, "test-agent") Expect(err).To(HaveOccurred()) var svcErr *service.ServiceError @@ -322,88 +177,58 @@ var _ = Describe("InstanceService", func() { errors.As(err, &svcErr) Expect(svcErr.Code).To(Equal(service.ErrCodeValidation)) Expect(svcErr.Message).To(ContainSubstring("must not be empty")) - Expect(providerCalled).To(BeFalse()) }) - It("returns validation error when spec.service_type is whitespace only", func() { + It("returns validation error when agentName is empty instead of creating an orphan pending row", func() { req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": " "}, + Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, } - _, err := instanceService.CreateInstance(ctx, req, nil) + _, err := instanceService.CreateInstance(ctx, req, nil, "") Expect(err).To(HaveOccurred()) var svcErr *service.ServiceError Expect(err).To(BeAssignableToTypeOf(svcErr)) errors.As(err, &svcErr) Expect(svcErr.Code).To(Equal(service.ErrCodeValidation)) - Expect(svcErr.Message).To(ContainSubstring("must not be empty")) - Expect(providerCalled).To(BeFalse()) - }) - - It("returns internal error with instance ID when DB insert fails", func() { - var instanceID string - var providerCallCount int - mockProviderWithID := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - providerCallCount++ - instanceID = uuid.New().String() + Expect(svcErr.Message).To(ContainSubstring("agent_name")) - if providerCallCount == 1 { - sqlDB, _ := db.DB() - _ = sqlDB.Close() - } - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - _ = json.NewEncoder(w).Encode(map[string]string{ - "id": instanceID, - "status": "PROVISIONING", - }) - })) - defer mockProviderWithID.Close() - - providerWithID := model.Provider{ - ID: uuid.New().String(), - Name: "provider-db-fail", - ServiceType: "vm", - Endpoint: mockProviderWithID.URL, - HealthStatus: model.HealthStatusReady, - } - Expect(db.Create(&providerWithID).Error).NotTo(HaveOccurred()) + var count int64 + Expect(db.Model(&model.ServiceTypeInstance{}).Count(&count).Error).NotTo(HaveOccurred()) + Expect(count).To(BeZero(), "no instance row should have been created") + }) + It("returns validation error when agentName is whitespace only", func() { req := &resource_manager.ServiceTypeInstance{ - ProviderName: "provider-db-fail", - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, + Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, } - _, err := instanceService.CreateInstance(ctx, req, nil) + _, err := instanceService.CreateInstance(ctx, req, nil, " ") Expect(err).To(HaveOccurred()) var svcErr *service.ServiceError Expect(err).To(BeAssignableToTypeOf(svcErr)) errors.As(err, &svcErr) - Expect(svcErr.Code).To(Equal(service.ErrCodeInternal)) - Expect(svcErr.Message).To(ContainSubstring("failed to create database record")) - Expect(svcErr.Message).To(ContainSubstring(instanceID)) + Expect(svcErr.Code).To(Equal(service.ErrCodeValidation)) }) }) Describe("GetInstance", func() { It("returns an instance", func() { - // Create an instance first - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "pending", + InstanceName: "get-inst", + Spec: map[string]any{"cpu": 2}, } - created, _ := instanceService.CreateInstance(ctx, req, nil) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) - result, err := instanceService.GetInstance(ctx, *created.Id, false) + result, err := instanceService.GetInstance(ctx, inst.ID, false) Expect(err).NotTo(HaveOccurred()) Expect(result).NotTo(BeNil()) - Expect(*result.Id).To(Equal(*created.Id)) - Expect(result.ProviderName).To(Equal("test-provider")) + Expect(*result.Id).To(Equal(inst.ID)) }) It("returns not found error for non-existent instance", func() { @@ -427,14 +252,15 @@ var _ = Describe("InstanceService", func() { }) It("returns all instances", func() { - // Create instances for i := 0; i < 3; i++ { - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": i + 1, "service_type": "vm"}, + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "pending", + InstanceName: uuid.New().String(), + Spec: map[string]any{"cpu": i + 1}, } - _, err := instanceService.CreateInstance(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) } result, err := instanceService.ListInstances(ctx, nil, nil, false, nil, nil) @@ -443,288 +269,397 @@ var _ = Describe("InstanceService", func() { Expect(*result.Instances).To(HaveLen(3)) }) - It("respects max page size and returns next page token", func() { - // Create 5 instances - for i := 0; i < 5; i++ { - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": i + 1, "service_type": "vm"}, - } - _, err := instanceService.CreateInstance(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) - } - - maxPageSize := 2 - result, err := instanceService.ListInstances(ctx, nil, nil, false, &maxPageSize, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(*result.Instances).To(HaveLen(2)) - Expect(result.NextPageToken).NotTo(BeNil()) - Expect(*result.NextPageToken).NotTo(BeEmpty()) - - // Get second page using token - secondPage, err := instanceService.ListInstances(ctx, nil, nil, false, &maxPageSize, result.NextPageToken) - - Expect(err).NotTo(HaveOccurred()) - Expect(*secondPage.Instances).To(HaveLen(2)) - Expect(secondPage.NextPageToken).NotTo(BeNil()) - - // Verify instances are different between pages - firstIDs := make(map[string]bool) - for _, inst := range *result.Instances { - firstIDs[*inst.Id] = true - } - for _, inst := range *secondPage.Instances { - Expect(firstIDs[*inst.Id]).To(BeFalse(), "Instance should not appear in both pages") - } - - // Get third page (last page with 1 item) - thirdPage, err := instanceService.ListInstances(ctx, nil, nil, false, &maxPageSize, secondPage.NextPageToken) - Expect(err).NotTo(HaveOccurred()) - Expect(*thirdPage.Instances).To(HaveLen(1)) - Expect(thirdPage.NextPageToken).To(BeNil()) - }) - It("filters instances by service type", func() { - containerProvider := model.Provider{ - ID: uuid.New().String(), - Name: "container-provider", - ServiceType: "container", - Endpoint: mockProvider.URL, - HealthStatus: model.HealthStatusReady, - } - Expect(db.Create(&containerProvider).Error).NotTo(HaveOccurred()) - - // Create vm instances with service_type in spec for i := 0; i < 2; i++ { - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": i + 1, "service_type": "vm"}, + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "pending", + InstanceName: uuid.New().String(), + Spec: map[string]any{"cpu": i + 1}, } - _, err := instanceService.CreateInstance(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) } - - // Create container instances with service_type in spec for i := 0; i < 3; i++ { - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "container-provider", - Spec: map[string]interface{}{"image": "nginx", "service_type": "container"}, + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "container", + Status: "pending", + InstanceName: uuid.New().String(), + Spec: map[string]any{"image": "nginx"}, } - _, err := instanceService.CreateInstance(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) } vmType := "vm" - result, err := instanceService.ListInstances(ctx, nil, &vmType, false, nil, nil) + result, err := instanceService.ListInstances(ctx, &vmType, nil, false, nil, nil) Expect(err).NotTo(HaveOccurred()) Expect(*result.Instances).To(HaveLen(2)) containerType := "container" - result, err = instanceService.ListInstances(ctx, nil, &containerType, false, nil, nil) + result, err = instanceService.ListInstances(ctx, &containerType, nil, false, nil, nil) Expect(err).NotTo(HaveOccurred()) Expect(*result.Instances).To(HaveLen(3)) }) - It("filters instances by provider name", func() { - // Create a second provider - secondProvider := model.Provider{ - ID: uuid.New().String(), - Name: "second-provider", - ServiceType: "vm", - Endpoint: mockProvider.URL, - HealthStatus: model.HealthStatusReady, - } - Expect(db.Create(&secondProvider).Error).NotTo(HaveOccurred()) - - // Create instances for different providers + It("filters instances by agent name", func() { + agentA, agentB := "agent-a", "agent-b" for i := 0; i < 2; i++ { - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": i + 1, "service_type": "vm"}, + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "pending", + InstanceName: uuid.New().String(), + Spec: map[string]any{"cpu": i + 1}, + AgentName: &agentA, } - _, err := instanceService.CreateInstance(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) } - - for i := 0; i < 3; i++ { - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "second-provider", - Spec: map[string]interface{}{"cpu": i + 1, "service_type": "vm"}, - } - _, err := instanceService.CreateInstance(ctx, req, nil) - Expect(err).NotTo(HaveOccurred()) + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "pending", + InstanceName: uuid.New().String(), + Spec: map[string]any{"cpu": 3}, + AgentName: &agentB, } + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) - // Filter by first provider - filterProvider := "test-provider" - result, err := instanceService.ListInstances(ctx, &filterProvider, nil, false, nil, nil) - + result, err := instanceService.ListInstances(ctx, nil, &agentA, false, nil, nil) Expect(err).NotTo(HaveOccurred()) Expect(*result.Instances).To(HaveLen(2)) - for _, inst := range *result.Instances { - Expect(inst.ProviderName).To(Equal("test-provider")) - } - - // Filter by second provider - filterProvider = "second-provider" - result, err = instanceService.ListInstances(ctx, &filterProvider, nil, false, nil, nil) + result, err = instanceService.ListInstances(ctx, nil, &agentB, false, nil, nil) Expect(err).NotTo(HaveOccurred()) - Expect(*result.Instances).To(HaveLen(3)) - for _, inst := range *result.Instances { - Expect(inst.ProviderName).To(Equal("second-provider")) - } + Expect(*result.Instances).To(HaveLen(1)) }) }) - Describe("DeleteInstance", func() { - It("deletes an instance", func() { - // Create an instance first - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, + Describe("DeleteInstance (agent-routed)", func() { + It("publishes delete event and marks deleting, awaiting agent acknowledgement, for non-deferred deletion", func() { + agentName := "test-agent" + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "running", + InstanceName: "del-inst", + Spec: map[string]any{"cpu": 2}, + AgentName: &agentName, } - created, _ := instanceService.CreateInstance(ctx, req, nil) - - err := instanceService.DeleteInstance(ctx, *created.Id, false) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + err := instanceService.DeleteInstance(ctx, inst.ID, false) Expect(err).NotTo(HaveOccurred()) - Expect(deleteRequested).To(BeTrue()) - // Verify it's deleted - _, err = instanceService.GetInstance(ctx, *created.Id, false) + // The record must still exist as "deleting" until the agent's + // deletion-acknowledged event confirms the physical resource is + // gone, and be enrolled in retry tracking like a deferred delete. + got, getErr := instanceService.GetInstance(ctx, inst.ID, true) + Expect(getErr).NotTo(HaveOccurred()) + Expect(got.Status).NotTo(BeNil()) + Expect(*got.Status).To(Equal("deleting")) + + _, hiddenErr := instanceService.GetInstance(ctx, inst.ID, false) var svcErr *service.ServiceError - Expect(err).To(BeAssignableToTypeOf(svcErr)) - errors.As(err, &svcErr) + Expect(hiddenErr).To(BeAssignableToTypeOf(svcErr)) + errors.As(hiddenErr, &svcErr) Expect(svcErr.Code).To(Equal(service.ErrCodeNotFound)) }) - It("returns not found error for non-existent instance", func() { - err := instanceService.DeleteInstance(ctx, uuid.New().String(), false) + It("hard-deletes immediately when the instance has no agent", func() { + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "running", + InstanceName: "del-inst-no-agent", + Spec: map[string]any{"cpu": 2}, + } + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) - Expect(err).To(HaveOccurred()) + err := instanceService.DeleteInstance(ctx, inst.ID, false) + Expect(err).NotTo(HaveOccurred()) + + _, getErr := instanceService.GetInstance(ctx, inst.ID, false) var svcErr *service.ServiceError - Expect(err).To(BeAssignableToTypeOf(svcErr)) - errors.As(err, &svcErr) + Expect(getErr).To(BeAssignableToTypeOf(svcErr)) + errors.As(getErr, &svcErr) Expect(svcErr.Code).To(Equal(service.ErrCodeNotFound)) }) - It("returns error when provider is missing and deferred is false", func() { - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, + It("hard-deletes immediately when the assigned agent no longer exists (C)", func() { + // Without this, publishDeleteToAgent's old behavior (silently + // treating ErrAgentNotFound as a successful publish) would leave + // this instance stuck in "deleting" forever: no agent will ever + // send a "deletion-acknowledged" for a nonexistent agent. + gone := "nonexistent-agent" + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "running", + InstanceName: "del-inst-gone-agent", + Spec: map[string]any{"cpu": 2}, + AgentName: &gone, } - created, _ := instanceService.CreateInstance(ctx, req, nil) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) - // Delete the provider from the database - Expect(db.Delete(&model.Provider{}, "name = ?", "test-provider").Error).NotTo(HaveOccurred()) + err := instanceService.DeleteInstance(ctx, inst.ID, false) + Expect(err).NotTo(HaveOccurred()) - err := instanceService.DeleteInstance(ctx, *created.Id, false) + _, getErr := instanceService.GetInstance(ctx, inst.ID, true) + var svcErr *service.ServiceError + Expect(getErr).To(BeAssignableToTypeOf(svcErr)) + errors.As(getErr, &svcErr) + Expect(svcErr.Code).To(Equal(service.ErrCodeNotFound)) + }) + + It("returns not found error for non-existent instance", func() { + err := instanceService.DeleteInstance(ctx, uuid.New().String(), false) Expect(err).To(HaveOccurred()) var svcErr *service.ServiceError Expect(err).To(BeAssignableToTypeOf(svcErr)) errors.As(err, &svcErr) - Expect(svcErr.Code).To(Equal(service.ErrCodeProviderError)) + Expect(svcErr.Code).To(Equal(service.ErrCodeNotFound)) }) It("defers deletion without contacting provider", func() { - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, + agentName := "test-agent" + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "running", + InstanceName: "defer-del-inst", + Spec: map[string]any{"cpu": 2}, + AgentName: &agentName, } - created, _ := instanceService.CreateInstance(ctx, req, nil) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) - // Deferred delete should succeed without calling the provider - err := instanceService.DeleteInstance(ctx, *created.Id, true) + err := instanceService.DeleteInstance(ctx, inst.ID, true) Expect(err).NotTo(HaveOccurred()) - Expect(deleteRequested).To(BeFalse()) - // Instance should be marked for deletion, not visible in default list result, err := instanceService.ListInstances(ctx, nil, nil, false, nil, nil) Expect(err).NotTo(HaveOccurred()) Expect(*result.Instances).To(BeEmpty()) - // But visible with show_deleted result, err = instanceService.ListInstances(ctx, nil, nil, true, nil, nil) Expect(err).NotTo(HaveOccurred()) Expect(*result.Instances).To(HaveLen(1)) Expect(string(*(*result.Instances)[0].DeletionStatus)).To(Equal("SCHEDULED")) }) + }) - It("defers deletion without contacting provider even when provider is missing", func() { - req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, + Describe("InstanceService agent fields", func() { + It("stores agent_name on instance record and surfaces it on the API struct (F19)", func() { + agentName := "test-agent" + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "pending", + InstanceName: "agent-inst", + Spec: map[string]any{"cpu": 2}, + AgentName: &agentName, } - created, _ := instanceService.CreateInstance(ctx, req, nil) + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) - // Delete the provider from the database - Expect(db.Delete(&model.Provider{}, "name = ?", "test-provider").Error).NotTo(HaveOccurred()) - - // Deferred delete should succeed without attempting provider call - err := instanceService.DeleteInstance(ctx, *created.Id, true) - Expect(err).NotTo(HaveOccurred()) - Expect(deleteRequested).To(BeFalse()) + var stored model.ServiceTypeInstance + Expect(db.First(&stored, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + Expect(stored.AgentName).NotTo(BeNil()) + Expect(*stored.AgentName).To(Equal("test-agent")) - // Verify marked as SCHEDULED - result, err := instanceService.ListInstances(ctx, nil, nil, true, nil, nil) + // ModelToAPI must populate AgentName on the returned API struct + // too, not just the DB row. + got, err := instanceService.GetInstance(ctx, inst.ID, false) Expect(err).NotTo(HaveOccurred()) - Expect(*result.Instances).To(HaveLen(1)) - Expect(string(*(*result.Instances)[0].DeletionStatus)).To(Equal("SCHEDULED")) + Expect(got.AgentName).NotTo(BeNil()) + Expect(*got.AgentName).To(Equal("test-agent")) }) + }) - It("resets retry count when deleting a FAILED instance with deferred=true", func() { + Describe("Agent validation", func() { + It("rejects creation when agent is not found", func() { req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, + Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, } - created, _ := instanceService.CreateInstance(ctx, req, nil) - // Manually mark instance as FAILED with high retry count - failed := "FAILED" - Expect(db.Model(&model.ServiceTypeInstance{}).Where("id = ?", *created.Id).Updates(map[string]interface{}{ - "deletion_status": failed, - "retry_count": 10, + _, err := instanceService.CreateInstance(ctx, req, nil, "nonexistent-agent") + + Expect(err).To(HaveOccurred()) + var svcErr *service.ServiceError + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeNotFound)) + }) + + It("rejects creation when agent is unavailable", func() { + Expect(db.Create(&agentmodel.Agent{ + ID: uuid.New().String(), + Name: "unavailable-agent", + TopicName: "dcm.agent.unavailable-agent", + HealthStatus: agentmodel.AgentHealthStatusUnavailable, + ServiceTypes: []string{"vm"}, }).Error).NotTo(HaveOccurred()) - // Deferred delete should succeed and reset retry count without contacting provider - err := instanceService.DeleteInstance(ctx, *created.Id, true) - Expect(err).NotTo(HaveOccurred()) - Expect(deleteRequested).To(BeFalse()) + req := &resource_manager.ServiceTypeInstance{ + Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, + } - // Verify retry count was reset and status is SCHEDULED - var instance model.ServiceTypeInstance - Expect(db.Where("id = ?", *created.Id).First(&instance).Error).NotTo(HaveOccurred()) - Expect(instance.RetryCount).To(Equal(0)) - Expect(*instance.DeletionStatus).To(Equal("SCHEDULED")) + _, err := instanceService.CreateInstance(ctx, req, nil, "unavailable-agent") + + Expect(err).To(HaveOccurred()) + var svcErr *service.ServiceError + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeUnavailable)) }) - It("returns 204 when deleting an already-pending instance and SP succeeds", func() { + It("rejects creation when agent is congested", func() { + Expect(db.Create(&agentmodel.Agent{ + ID: uuid.New().String(), + Name: "congested-agent", + TopicName: "dcm.agent.congested-agent", + HealthStatus: agentmodel.AgentHealthStatusCongested, + ServiceTypes: []string{"vm"}, + }).Error).NotTo(HaveOccurred()) + req := &resource_manager.ServiceTypeInstance{ - ProviderName: "test-provider", - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, + Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, } - created, _ := instanceService.CreateInstance(ctx, req, nil) - // Manually mark instance as SCHEDULED - scheduled := "SCHEDULED" - Expect(db.Model(&model.ServiceTypeInstance{}).Where("id = ?", *created.Id).Updates(map[string]interface{}{ - "deletion_status": scheduled, + _, err := instanceService.CreateInstance(ctx, req, nil, "congested-agent") + + Expect(err).To(HaveOccurred()) + var svcErr *service.ServiceError + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeUnavailable)) + }) + + It("rejects creation when agent does not serve the requested service type", func() { + Expect(db.Create(&agentmodel.Agent{ + ID: uuid.New().String(), + Name: "container-only-agent", + TopicName: "dcm.agent.container-only-agent", + HealthStatus: agentmodel.AgentHealthStatusReady, + ServiceTypes: []string{"container"}, }).Error).NotTo(HaveOccurred()) - // Delete should succeed (SP is available) and hard-delete the record - err := instanceService.DeleteInstance(ctx, *created.Id, false) + req := &resource_manager.ServiceTypeInstance{ + Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, + } + + _, err := instanceService.CreateInstance(ctx, req, nil, "container-only-agent") + + Expect(err).To(HaveOccurred()) + var svcErr *service.ServiceError + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeValidation)) + Expect(svcErr.Message).To(ContainSubstring("does not serve service type")) + }) + + It("accepts creation when agent is ready and serves the service type", func() { + req := &resource_manager.ServiceTypeInstance{ + Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, + } + + result, err := instanceService.CreateInstance(ctx, req, nil, "test-agent") + Expect(err).NotTo(HaveOccurred()) - Expect(deleteRequested).To(BeTrue()) + Expect(result).NotTo(BeNil()) + }) + }) + + Describe("ReassignAgent", func() { + BeforeEach(func() { + Expect(db.Create(&agentmodel.Agent{ID: uuid.New().String(), Name: "fallback-agent", TopicName: "dcm.agent.fallback-agent", HealthStatus: agentmodel.AgentHealthStatusReady, ServiceTypes: []string{"vm"}}).Error).NotTo(HaveOccurred()) + }) + + It("reassigns when expectedCurrentAgent matches the instance's current agent", func() { + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "pending", + InstanceName: "reassign-cas-match", + Spec: map[string]any{"cpu": 2}, + AgentName: ptrString("test-agent"), + } + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + err := instanceService.ReassignAgent(ctx, inst.ID, "fallback-agent", "test-agent") + + Expect(err).NotTo(HaveOccurred()) + var stored model.ServiceTypeInstance + Expect(db.First(&stored, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + Expect(*stored.AgentName).To(Equal("fallback-agent")) + }) + + It("rejects the reassignment when expectedCurrentAgent is stale (R2 T1: CAS parameter must be threaded end-to-end, not re-derived from a fresh read)", func() { + // Proves expectedCurrentAgent actually reaches ReassignAndReset's + // CAS rather than being silently overridden by a fresh Get() + // inside ReassignAgent, which would defeat the whole guard: a + // caller passing a stale/excluded agent it observed earlier + // must be rejected here even though the DB's current agent_name + // ("test-agent") looks otherwise eligible (pending, not deleted). + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "pending", + InstanceName: "reassign-cas-stale", + Spec: map[string]any{"cpu": 2}, + AgentName: ptrString("test-agent"), + } + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + err := instanceService.ReassignAgent(ctx, inst.ID, "fallback-agent", "some-other-agent-the-caller-thinks-is-current") - // Verify it's fully gone - _, err = instanceService.GetInstance(ctx, *created.Id, false) + Expect(err).To(HaveOccurred()) var svcErr *service.ServiceError - Expect(err).To(BeAssignableToTypeOf(svcErr)) - errors.As(err, &svcErr) - Expect(svcErr.Code).To(Equal(service.ErrCodeNotFound)) + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeConflict)) + + var stored model.ServiceTypeInstance + Expect(db.First(&stored, "id = ?", inst.ID).Error).NotTo(HaveOccurred()) + Expect(*stored.AgentName).To(Equal("test-agent")) + }) + + It("returns validation error when agentName is empty", func() { + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "pending", + InstanceName: "reassign-empty-agent", + Spec: map[string]any{"cpu": 2}, + } + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + err := instanceService.ReassignAgent(ctx, inst.ID, "", "") + + Expect(err).To(HaveOccurred()) + var svcErr *service.ServiceError + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeValidation)) + }) + + It("returns unavailable error when agent store is not configured", func() { + // Regression test: ReassignAgent previously had no guard of its + // own before calling validateAgent (unlike CreateInstance), and + // validateAgent's nil-agentStore check used to silently skip + // validation (return nil) instead of erroring. A nil agentStore + // must fail fast here, not be treated as "agent is valid". + inst := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "pending", + InstanceName: "reassign-no-agent-store", + Spec: map[string]any{"cpu": 2}, + } + Expect(db.Create(&inst).Error).NotTo(HaveOccurred()) + + pub := messaging.NewPublisher(&stubJetStream{}) + noAgentStoreService := rmsvc.NewInstanceService(dataStore, pub, nil) + + err := noAgentStoreService.ReassignAgent(ctx, inst.ID, "test-agent", "") + + Expect(err).To(HaveOccurred()) + var svcErr *service.ServiceError + Expect(errors.As(err, &svcErr)).To(BeTrue()) + Expect(svcErr.Code).To(Equal(service.ErrCodeUnavailable)) }) }) }) diff --git a/internal/sp/store/db.go b/internal/sp/store/db.go index 220135b..1ceb6c9 100644 --- a/internal/sp/store/db.go +++ b/internal/sp/store/db.go @@ -64,7 +64,7 @@ func InitDB(cfg *config.Config) (*gorm.DB, error) { slog.Info("Database connection established", "type", cfg.Database.Type) - if err := db.AutoMigrate(&model.Provider{}, &model.ServiceTypeInstance{}); err != nil { + if err := db.AutoMigrate(&model.ServiceTypeInstance{}); err != nil { return nil, fmt.Errorf("failed to migrate database: %w", err) } diff --git a/internal/sp/store/model/provider.go b/internal/sp/store/model/provider.go deleted file mode 100644 index ddf01fb..0000000 --- a/internal/sp/store/model/provider.go +++ /dev/null @@ -1,43 +0,0 @@ -// Package model defines database models used by the store layer. -package model - -import ( - "time" -) - -// HealthStatus represents the health status of a provider -type HealthStatus string - -const ( - // HealthStatusReady indicates the provider is healthy and ready to serve requests - HealthStatusReady HealthStatus = "ready" - // HealthStatusUnhealthy indicates the provider is reachable but the backing provider is unavailable - HealthStatusUnhealthy HealthStatus = "unhealthy" - // HealthStatusUnavailable indicates the provider is unreachable - HealthStatusUnavailable HealthStatus = "unavailable" -) - -func (h HealthStatus) StringPtr() *string { - s := string(h) - return &s -} - -type Provider struct { - ID string `gorm:"primaryKey;type:varchar(63)"` - Name string `gorm:"uniqueIndex;not null"` - ServiceType string `gorm:"column:service_type;not null"` - SchemaVersion string `gorm:"column:schema_version;not null"` - Endpoint string `gorm:"column:endpoint;not null"` - DisplayName *string `gorm:"column:display_name"` - Operations []string `gorm:"column:operations;serializer:json"` - Metadata map[string]interface{} `gorm:"column:metadata;serializer:json"` - CreateTime time.Time `gorm:"column:create_time;autoCreateTime"` - UpdateTime time.Time `gorm:"column:update_time;autoUpdateTime"` - - // Health check fields - HealthStatus HealthStatus `gorm:"column:health_status;default:ready"` - ConsecutiveFailures int `gorm:"column:consecutive_failures;default:0"` - NextHealthCheck *time.Time `gorm:"column:next_health_check"` -} - -type ProviderList []Provider diff --git a/internal/sp/store/model/service_type_instance.go b/internal/sp/store/model/service_type_instance.go index ebf1923..bf600ff 100644 --- a/internal/sp/store/model/service_type_instance.go +++ b/internal/sp/store/model/service_type_instance.go @@ -1,3 +1,4 @@ +// Package model defines database models for the service provider store. package model import ( @@ -5,18 +6,35 @@ import ( ) type ServiceTypeInstance struct { - ID string `gorm:"primaryKey;type:varchar(63)"` - ProviderName string `gorm:"column:provider_name;not null"` - ServiceType string `gorm:"column:service_type;not null;default:'';index"` - Status string `gorm:"column:status;not null"` + ID string `gorm:"primaryKey;type:varchar(63)"` + ServiceType string `gorm:"column:service_type;not null;default:'';index"` + // idx_sti_status_pending backs the periodic sweepPending/sweepQueued + // hot-path queries (status = ? AND pending_started_at < ?), which + // without it would degrade to a full table scan on every sweep tick as + // the instance count grows. + Status string `gorm:"column:status;not null;index:idx_sti_status_pending,priority:1"` StatusMessage string `gorm:"column:status_message"` InstanceName string `gorm:"column:instance_name;not null"` Spec map[string]any `gorm:"column:spec;type:jsonb;serializer:json;not null"` CreateTime time.Time `gorm:"column:create_time;autoCreateTime"` UpdateTime time.Time `gorm:"column:update_time;autoUpdateTime"` + // AgentName is a plain string reference to the owning agent's natural key. + // Intentionally NOT a GORM foreign key: agents can be deregistered while + // instances still exist (orphans are tolerated and resolved by the + // cleanup scheduler). Application-level validation (see + // InstanceService.validateAgent) enforces the agent exists and is ready + // before this field is ever set. + AgentName *string `gorm:"column:agent_name"` + PendingStartedAt *time.Time `gorm:"column:pending_started_at;index:idx_sti_status_pending,priority:2"` + // Soft-delete fields for deferred deletion (rehydration flow) - DeletionStatus *string `gorm:"column:deletion_status"` + // + // deletion_status is indexed for ListPendingDeletions (WHERE + // deletion_status = 'SCHEDULED') and the default Get/List visibility + // filter (WHERE deletion_status IS NULL), both hot paths run on every + // cleanup-scheduler tick and API list call respectively. + DeletionStatus *string `gorm:"column:deletion_status;index"` RetryCount int `gorm:"column:retry_count;default:0"` LastDeletionAttempt *time.Time `gorm:"column:last_deletion_attempt"` DeletionRequestedAt *time.Time `gorm:"column:deletion_requested_at"` diff --git a/internal/sp/store/model/status.go b/internal/sp/store/model/status.go new file mode 100644 index 0000000..2910e17 --- /dev/null +++ b/internal/sp/store/model/status.go @@ -0,0 +1,21 @@ +package model + +// Status values for ServiceTypeInstance.Status: the canonical lowercase +// lifecycle states written by the agent-based provisioning flow (create +// service, sweep, response consumer). Kept as constants so the states used +// across packages can't drift in casing. +// +// The legacy provider StatusConsumer (internal/sp/consumer/consumer.go) +// receives status strings from external CloudEvent producers it does not +// control; it normalizes incoming values to this lowercase convention at +// ingestion rather than trusting upstream casing. +const ( + StatusPending = "pending" + StatusQueued = "queued" + StatusProvisioning = "provisioning" + StatusRunning = "running" + StatusDeleting = "deleting" + StatusCancelled = "cancelled" + StatusFailed = "failed" + StatusPendingDeletion = "pending_deletion" +) diff --git a/internal/sp/store/provider/provider.go b/internal/sp/store/provider/provider.go deleted file mode 100644 index b4db5f9..0000000 --- a/internal/sp/store/provider/provider.go +++ /dev/null @@ -1,196 +0,0 @@ -// Package provider provides database access for provider operations. -package provider - -import ( - "context" - "errors" - "time" - - "github.com/dcm-project/control-plane/internal/sp/store/model" - "gorm.io/gorm" - "gorm.io/gorm/clause" -) - -var ( - ErrProviderNotFound = errors.New("provider not found") - ErrProviderNameTaken = errors.New("provider name already taken") -) - -// ProviderFilter contains optional fields for filtering provider queries. -// nil fields are ignored (not filtered). -type ProviderFilter struct { - Name *string - ServiceType *string -} - -// Pagination contains options for paginated queries. -type Pagination struct { - Limit int - Offset int -} - -type Provider interface { - List(ctx context.Context, filter *ProviderFilter, pagination *Pagination) (model.ProviderList, error) - Count(ctx context.Context, filter *ProviderFilter) (int64, error) - Create(ctx context.Context, provider model.Provider) (*model.Provider, error) - Delete(ctx context.Context, id string) error - Update(ctx context.Context, provider model.Provider) (*model.Provider, error) - Get(ctx context.Context, id string) (*model.Provider, error) - GetByName(ctx context.Context, name string) (*model.Provider, error) - ExistsByID(ctx context.Context, id string) (bool, error) - - // Health check methods - ListProvidersForHealthCheck(ctx context.Context, now time.Time) (model.ProviderList, error) - UpdateHealthStatus(ctx context.Context, id string, status model.HealthStatus, consecutiveFailures int, nextCheck time.Time) error -} - -type ProviderStore struct { - db *gorm.DB -} - -var _ Provider = (*ProviderStore)(nil) - -func NewProvider(db *gorm.DB) Provider { - return &ProviderStore{db: db} -} - -func (s *ProviderStore) List(ctx context.Context, filter *ProviderFilter, pagination *Pagination) (model.ProviderList, error) { - var providers model.ProviderList - query := s.db.WithContext(ctx) - - if filter != nil { - if filter.Name != nil { - query = query.Where(&model.Provider{Name: *filter.Name}) - } - if filter.ServiceType != nil { - query = query.Where(&model.Provider{ServiceType: *filter.ServiceType}) - } - } - - // Apply consistent ordering for pagination - query = query.Order("create_time ASC, id ASC") - - if pagination != nil { - query = query.Limit(pagination.Limit).Offset(pagination.Offset) - } - - if err := query.Find(&providers).Error; err != nil { - return nil, err - } - return providers, nil -} - -func (s *ProviderStore) Count(ctx context.Context, filter *ProviderFilter) (int64, error) { - var count int64 - query := s.db.WithContext(ctx).Model(&model.Provider{}) - - if filter != nil { - if filter.Name != nil { - query = query.Where(&model.Provider{Name: *filter.Name}) - } - if filter.ServiceType != nil { - query = query.Where(&model.Provider{ServiceType: *filter.ServiceType}) - } - } - - if err := query.Count(&count).Error; err != nil { - return 0, err - } - return count, nil -} - -func (s *ProviderStore) Create(ctx context.Context, provider model.Provider) (*model.Provider, error) { - if err := s.db.WithContext(ctx).Clauses(clause.Returning{}).Create(&provider).Error; err != nil { - return nil, err - } - return &provider, nil -} - -func (s *ProviderStore) Delete(ctx context.Context, id string) error { - result := s.db.WithContext(ctx).Where("id = ?", id).Delete(&model.Provider{}) - if result.Error != nil { - return result.Error - } - if result.RowsAffected == 0 { - return ErrProviderNotFound - } - return nil -} - -func (s *ProviderStore) Update(ctx context.Context, provider model.Provider) (*model.Provider, error) { - // Select explicit columns so nullable JSON and *string fields can be cleared (not skipped as zero). - result := s.db.WithContext(ctx).Model(&provider).Clauses(clause.Returning{}). - Select( - "Name", "ServiceType", "SchemaVersion", "Endpoint", - "DisplayName", "Operations", "Metadata", "UpdateTime", - "ConsecutiveFailures", "NextHealthCheck", - ).Updates(&provider) - if result.Error != nil { - return nil, result.Error - } - if result.RowsAffected == 0 { - return nil, ErrProviderNotFound - } - return &provider, nil -} - -func (s *ProviderStore) Get(ctx context.Context, id string) (*model.Provider, error) { - var provider model.Provider - if err := s.db.WithContext(ctx).Where("id = ?", id).First(&provider).Error; err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, ErrProviderNotFound - } - return nil, err - } - return &provider, nil -} - -func (s *ProviderStore) GetByName(ctx context.Context, name string) (*model.Provider, error) { - var provider model.Provider - if err := s.db.WithContext(ctx).Where(&model.Provider{Name: name}).First(&provider).Error; err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, ErrProviderNotFound - } - return nil, err - } - return &provider, nil -} - -func (s *ProviderStore) ExistsByID(ctx context.Context, id string) (bool, error) { - var provider model.Provider - err := s.db.WithContext(ctx).Select("id").Where(&model.Provider{ID: id}).Take(&provider).Error - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return false, nil - } - return false, err - } - return true, nil -} - -// ListProvidersForHealthCheck returns providers that are due for a health check. -func (s *ProviderStore) ListProvidersForHealthCheck(ctx context.Context, now time.Time) (model.ProviderList, error) { - var providers model.ProviderList - if err := s.db.WithContext(ctx). - Where("next_health_check IS NULL OR next_health_check <= ?", now). - Find(&providers).Error; err != nil { - return nil, err - } - return providers, nil -} - -// UpdateHealthStatus updates the health status and tracking fields for a provider. -func (s *ProviderStore) UpdateHealthStatus(ctx context.Context, id string, status model.HealthStatus, consecutiveFailures int, nextCheck time.Time) error { - result := s.db.WithContext(ctx).Model(&model.Provider{}).Where("id = ?", id).Updates(map[string]interface{}{ - "health_status": status, - "consecutive_failures": consecutiveFailures, - "next_health_check": nextCheck, - }) - if result.Error != nil { - return result.Error - } - if result.RowsAffected == 0 { - return ErrProviderNotFound - } - return nil -} diff --git a/internal/sp/store/provider/provider_test.go b/internal/sp/store/provider/provider_test.go deleted file mode 100644 index 08cd6a9..0000000 --- a/internal/sp/store/provider/provider_test.go +++ /dev/null @@ -1,449 +0,0 @@ -package provider_test - -import ( - "context" - "time" - - "github.com/dcm-project/control-plane/internal/sp/store/model" - "github.com/dcm-project/control-plane/internal/sp/store/provider" - "github.com/google/uuid" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "gorm.io/driver/sqlite" - "gorm.io/gorm" - "gorm.io/gorm/logger" -) - -var _ = Describe("Provider Store", func() { - var ( - db *gorm.DB - providerStore provider.Provider - ctx context.Context - ) - - BeforeEach(func() { - var err error - db, err = gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ - Logger: logger.Default.LogMode(logger.Silent), - }) - Expect(err).NotTo(HaveOccurred()) - Expect(db.AutoMigrate(&model.Provider{})).To(Succeed()) - - providerStore = provider.NewProvider(db) - ctx = context.Background() - }) - - AfterEach(func() { - sqlDB, _ := db.DB() - _ = sqlDB.Close() - }) - - Describe("Create", func() { - It("persists the provider", func() { - p := newProvider("create-test") - created, err := providerStore.Create(ctx, p) - - Expect(err).NotTo(HaveOccurred()) - Expect(created.ID).To(Equal(p.ID)) - Expect(created.Name).To(Equal("create-test")) - Expect(created.SchemaVersion).To(Equal("v1alpha1")) - }) - - It("rejects duplicate names", func() { - p1 := newProvider("duplicate-name") - _, err := providerStore.Create(ctx, p1) - Expect(err).NotTo(HaveOccurred()) - - p2 := newProvider("duplicate-name") - _, err = providerStore.Create(ctx, p2) - Expect(err).To(HaveOccurred()) - }) - }) - - Describe("Get", func() { - It("retrieves by ID", func() { - p := newProvider("get-test") - _, err := providerStore.Create(ctx, p) - Expect(err).NotTo(HaveOccurred()) - - found, err := providerStore.Get(ctx, p.ID) - - Expect(err).NotTo(HaveOccurred()) - Expect(found.Name).To(Equal("get-test")) - }) - - It("returns ErrProviderNotFound for missing ID", func() { - _, err := providerStore.Get(ctx, uuid.New().String()) - - Expect(err).To(Equal(provider.ErrProviderNotFound)) - }) - }) - - Describe("GetByName", func() { - It("retrieves by name", func() { - p := newProvider("named-provider") - _, err := providerStore.Create(ctx, p) - Expect(err).NotTo(HaveOccurred()) - - found, err := providerStore.GetByName(ctx, "named-provider") - - Expect(err).NotTo(HaveOccurred()) - Expect(found.ID).To(Equal(p.ID)) - }) - - It("returns ErrProviderNotFound for missing name", func() { - _, err := providerStore.GetByName(ctx, "non-existent") - - Expect(err).To(Equal(provider.ErrProviderNotFound)) - }) - }) - - Describe("List", func() { - It("returns all providers when filter is nil", func() { - _, err := providerStore.Create(ctx, newProvider("p1")) - Expect(err).NotTo(HaveOccurred()) - _, err = providerStore.Create(ctx, newProvider("p2")) - Expect(err).NotTo(HaveOccurred()) - - providers, err := providerStore.List(ctx, nil, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(providers).To(HaveLen(2)) - }) - - It("filters by service type", func() { - p1 := newProvider("vm-provider") - p1.ServiceType = "vm" - _, err := providerStore.Create(ctx, p1) - Expect(err).NotTo(HaveOccurred()) - - p2 := newProvider("container-provider") - p2.ServiceType = "container" - _, err = providerStore.Create(ctx, p2) - Expect(err).NotTo(HaveOccurred()) - - vmType := "vm" - vms, err := providerStore.List(ctx, &provider.ProviderFilter{ServiceType: &vmType}, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(vms).To(HaveLen(1)) - Expect(vms[0].Name).To(Equal("vm-provider")) - }) - - It("filters by name", func() { - _, err := providerStore.Create(ctx, newProvider("find-me")) - Expect(err).NotTo(HaveOccurred()) - _, err = providerStore.Create(ctx, newProvider("not-me")) - Expect(err).NotTo(HaveOccurred()) - - name := "find-me" - providers, err := providerStore.List(ctx, &provider.ProviderFilter{Name: &name}, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(providers).To(HaveLen(1)) - Expect(providers[0].Name).To(Equal("find-me")) - }) - - It("filters by both name and service type", func() { - p1 := newProvider("vm-one") - p1.ServiceType = "vm" - _, err := providerStore.Create(ctx, p1) - Expect(err).NotTo(HaveOccurred()) - - p2 := newProvider("vm-two") - p2.ServiceType = "vm" - _, err = providerStore.Create(ctx, p2) - Expect(err).NotTo(HaveOccurred()) - - name := "vm-one" - vmType := "vm" - providers, err := providerStore.List(ctx, &provider.ProviderFilter{Name: &name, ServiceType: &vmType}, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(providers).To(HaveLen(1)) - Expect(providers[0].Name).To(Equal("vm-one")) - }) - - It("respects pagination limit", func() { - _, err := providerStore.Create(ctx, newProvider("page-p1")) - Expect(err).NotTo(HaveOccurred()) - _, err = providerStore.Create(ctx, newProvider("page-p2")) - Expect(err).NotTo(HaveOccurred()) - _, err = providerStore.Create(ctx, newProvider("page-p3")) - Expect(err).NotTo(HaveOccurred()) - - providers, err := providerStore.List(ctx, nil, &provider.Pagination{Limit: 2, Offset: 0}) - - Expect(err).NotTo(HaveOccurred()) - Expect(providers).To(HaveLen(2)) - }) - - It("respects pagination offset", func() { - _, err := providerStore.Create(ctx, newProvider("offset-p1")) - Expect(err).NotTo(HaveOccurred()) - _, err = providerStore.Create(ctx, newProvider("offset-p2")) - Expect(err).NotTo(HaveOccurred()) - _, err = providerStore.Create(ctx, newProvider("offset-p3")) - Expect(err).NotTo(HaveOccurred()) - - providers, err := providerStore.List(ctx, nil, &provider.Pagination{Limit: 10, Offset: 2}) - - Expect(err).NotTo(HaveOccurred()) - Expect(providers).To(HaveLen(1)) - }) - }) - - Describe("Count", func() { - It("returns total count without filter", func() { - _, err := providerStore.Create(ctx, newProvider("count-p1")) - Expect(err).NotTo(HaveOccurred()) - _, err = providerStore.Create(ctx, newProvider("count-p2")) - Expect(err).NotTo(HaveOccurred()) - - count, err := providerStore.Count(ctx, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(count).To(Equal(int64(2))) - }) - - It("returns filtered count", func() { - p1 := newProvider("count-vm") - p1.ServiceType = "vm" - _, err := providerStore.Create(ctx, p1) - Expect(err).NotTo(HaveOccurred()) - - p2 := newProvider("count-container") - p2.ServiceType = "container" - _, err = providerStore.Create(ctx, p2) - Expect(err).NotTo(HaveOccurred()) - - vmType := "vm" - count, err := providerStore.Count(ctx, &provider.ProviderFilter{ServiceType: &vmType}) - - Expect(err).NotTo(HaveOccurred()) - Expect(count).To(Equal(int64(1))) - }) - }) - - Describe("Delete", func() { - It("removes the provider", func() { - p := newProvider("to-delete") - _, err := providerStore.Create(ctx, p) - Expect(err).NotTo(HaveOccurred()) - - err = providerStore.Delete(ctx, p.ID) - - Expect(err).NotTo(HaveOccurred()) - }) - - It("returns ErrProviderNotFound for missing ID", func() { - err := providerStore.Delete(ctx, uuid.New().String()) - - Expect(err).To(Equal(provider.ErrProviderNotFound)) - }) - }) - - Describe("Update", func() { - It("modifies existing provider", func() { - p := newProvider("to-update") - _, err := providerStore.Create(ctx, p) - Expect(err).NotTo(HaveOccurred()) - - p.Endpoint = "https://new-endpoint.com" - updated, err := providerStore.Update(ctx, p) - - Expect(err).NotTo(HaveOccurred()) - Expect(updated.Endpoint).To(Equal("https://new-endpoint.com")) - }) - - It("returns ErrProviderNotFound for non-existing provider", func() { - p := newProvider("non-existing") - _, err := providerStore.Update(ctx, p) - - Expect(err).To(Equal(provider.ErrProviderNotFound)) - }) - }) - - Describe("ListProvidersForHealthCheck", func() { - It("returns providers with null next_health_check", func() { - p := newProvider("null-next-check") - p.NextHealthCheck = nil - _, err := providerStore.Create(ctx, p) - Expect(err).NotTo(HaveOccurred()) - - now := time.Now() - providers, err := providerStore.ListProvidersForHealthCheck(ctx, now) - - Expect(err).NotTo(HaveOccurred()) - Expect(providers).To(HaveLen(1)) - Expect(providers[0].Name).To(Equal("null-next-check")) - }) - - It("returns providers with next_health_check in the past", func() { - p := newProvider("past-check") - pastTime := time.Now().Add(-1 * time.Hour) - p.NextHealthCheck = &pastTime - _, err := providerStore.Create(ctx, p) - Expect(err).NotTo(HaveOccurred()) - - now := time.Now() - providers, err := providerStore.ListProvidersForHealthCheck(ctx, now) - - Expect(err).NotTo(HaveOccurred()) - Expect(providers).To(HaveLen(1)) - Expect(providers[0].Name).To(Equal("past-check")) - }) - - It("returns providers with next_health_check equal to now", func() { - now := time.Now() - p := newProvider("equal-check") - p.NextHealthCheck = &now - _, err := providerStore.Create(ctx, p) - Expect(err).NotTo(HaveOccurred()) - - providers, err := providerStore.ListProvidersForHealthCheck(ctx, now) - - Expect(err).NotTo(HaveOccurred()) - Expect(providers).To(HaveLen(1)) - Expect(providers[0].Name).To(Equal("equal-check")) - }) - - It("excludes providers with next_health_check in the future", func() { - p := newProvider("future-check") - futureTime := time.Now().Add(1 * time.Hour) - p.NextHealthCheck = &futureTime - _, err := providerStore.Create(ctx, p) - Expect(err).NotTo(HaveOccurred()) - - now := time.Now() - providers, err := providerStore.ListProvidersForHealthCheck(ctx, now) - - Expect(err).NotTo(HaveOccurred()) - Expect(providers).To(BeEmpty()) - }) - - It("returns empty list when no providers are due", func() { - p := newProvider("not-due") - futureTime := time.Now().Add(24 * time.Hour) - p.NextHealthCheck = &futureTime - _, err := providerStore.Create(ctx, p) - Expect(err).NotTo(HaveOccurred()) - - now := time.Now() - providers, err := providerStore.ListProvidersForHealthCheck(ctx, now) - - Expect(err).NotTo(HaveOccurred()) - Expect(providers).To(BeEmpty()) - }) - - It("returns multiple providers due for health check", func() { - p1 := newProvider("due-1") - p1.NextHealthCheck = nil - _, err := providerStore.Create(ctx, p1) - Expect(err).NotTo(HaveOccurred()) - - p2 := newProvider("due-2") - pastTime := time.Now().Add(-30 * time.Minute) - p2.NextHealthCheck = &pastTime - _, err = providerStore.Create(ctx, p2) - Expect(err).NotTo(HaveOccurred()) - - p3 := newProvider("not-due") - futureTime := time.Now().Add(1 * time.Hour) - p3.NextHealthCheck = &futureTime - _, err = providerStore.Create(ctx, p3) - Expect(err).NotTo(HaveOccurred()) - - now := time.Now() - providers, err := providerStore.ListProvidersForHealthCheck(ctx, now) - - Expect(err).NotTo(HaveOccurred()) - Expect(providers).To(HaveLen(2)) - }) - }) - - Describe("UpdateHealthStatus", func() { - It("updates health status to not_ready", func() { - p := newProvider("health-update") - _, err := providerStore.Create(ctx, p) - Expect(err).NotTo(HaveOccurred()) - - nextCheck := time.Now().Add(1 * time.Hour) - err = providerStore.UpdateHealthStatus(ctx, p.ID, model.HealthStatusUnavailable, 3, nextCheck) - - Expect(err).NotTo(HaveOccurred()) - - updated, err := providerStore.Get(ctx, p.ID) - Expect(err).NotTo(HaveOccurred()) - Expect(updated.HealthStatus).To(Equal(model.HealthStatusUnavailable)) - Expect(updated.ConsecutiveFailures).To(Equal(3)) - Expect(updated.NextHealthCheck).NotTo(BeNil()) - }) - - It("updates health status to ready", func() { - p := newProvider("health-ready") - p.HealthStatus = model.HealthStatusUnavailable - p.ConsecutiveFailures = 5 - _, err := providerStore.Create(ctx, p) - Expect(err).NotTo(HaveOccurred()) - - nextCheck := time.Now().Add(10 * time.Second) - err = providerStore.UpdateHealthStatus(ctx, p.ID, model.HealthStatusReady, 0, nextCheck) - - Expect(err).NotTo(HaveOccurred()) - - updated, err := providerStore.Get(ctx, p.ID) - Expect(err).NotTo(HaveOccurred()) - Expect(updated.HealthStatus).To(Equal(model.HealthStatusReady)) - Expect(updated.ConsecutiveFailures).To(Equal(0)) - }) - - It("updates consecutive failures count", func() { - p := newProvider("failure-count") - _, err := providerStore.Create(ctx, p) - Expect(err).NotTo(HaveOccurred()) - - nextCheck := time.Now().Add(30 * time.Second) - err = providerStore.UpdateHealthStatus(ctx, p.ID, model.HealthStatusReady, 2, nextCheck) - - Expect(err).NotTo(HaveOccurred()) - - updated, err := providerStore.Get(ctx, p.ID) - Expect(err).NotTo(HaveOccurred()) - Expect(updated.ConsecutiveFailures).To(Equal(2)) - }) - - It("updates next health check time", func() { - p := newProvider("next-check-update") - _, err := providerStore.Create(ctx, p) - Expect(err).NotTo(HaveOccurred()) - - nextCheck := time.Now().Add(5 * time.Minute) - err = providerStore.UpdateHealthStatus(ctx, p.ID, model.HealthStatusReady, 0, nextCheck) - - Expect(err).NotTo(HaveOccurred()) - - updated, err := providerStore.Get(ctx, p.ID) - Expect(err).NotTo(HaveOccurred()) - Expect(updated.NextHealthCheck).NotTo(BeNil()) - Expect(updated.NextHealthCheck.Unix()).To(Equal(nextCheck.Unix())) - }) - - It("returns ErrProviderNotFound for missing ID", func() { - nextCheck := time.Now().Add(1 * time.Hour) - err := providerStore.UpdateHealthStatus(ctx, uuid.New().String(), model.HealthStatusReady, 0, nextCheck) - - Expect(err).To(Equal(provider.ErrProviderNotFound)) - }) - }) -}) - -func newProvider(name string) model.Provider { - return model.Provider{ - ID: uuid.New().String(), - Name: name, - ServiceType: "vm", - SchemaVersion: "v1alpha1", - Endpoint: "https://example.com/api", - } -} diff --git a/internal/sp/store/resource_manager/service_instance.go b/internal/sp/store/resource_manager/service_instance.go index d54b4e7..b180810 100644 --- a/internal/sp/store/resource_manager/service_instance.go +++ b/internal/sp/store/resource_manager/service_instance.go @@ -15,15 +15,22 @@ import ( "gorm.io/gorm/clause" ) -var ErrInstanceNotFound = errors.New("service type instance not found") +var ( + ErrInstanceNotFound = errors.New("service type instance not found") + // ErrInstanceNotEligible is returned by ReassignAndReset when the + // instance exists but is mid-deletion (status "deleting" or + // "pending_deletion"), so a self-heal/reassignment can't resurrect it + // into "pending" out from under an in-flight delete. + ErrInstanceNotEligible = errors.New("instance is not eligible for reassignment") +) // ServiceTypeInstanceListOptions contains optional fields for listing instances. type ServiceTypeInstanceListOptions struct { - ProviderName *string - ServiceType *string - ShowDeleted bool - PageSize int - PageToken *string + ServiceType *string + AgentName *string + ShowDeleted bool + PageSize int + PageToken *string } // ServiceTypeInstanceListResult contains the result of a List operation. @@ -38,35 +45,48 @@ type ServiceTypeInstance interface { //nolint:interfacebloat Get(ctx context.Context, id string, showDeleted bool) (*model.ServiceTypeInstance, error) ExistsByID(ctx context.Context, id string) (bool, error) UpdateStatus(ctx context.Context, instanceID string, status string, statusMessage string) error + UpdateStatusFrom(ctx context.Context, instanceID string, fromStatuses []string, agentName string, status string, statusMessage string) (bool, error) + MarkQueued(ctx context.Context, id string, agentName string) error + ReassignAndReset(ctx context.Context, id string, agentName string, expectedCurrentAgent string) error MarkForDeletion(ctx context.Context, id string) error ListPendingDeletions(ctx context.Context) ([]model.ServiceTypeInstance, error) IncrementDeletionRetry(ctx context.Context, id string) error MarkDeletionFailed(ctx context.Context, id string) error + MarkDeletionComplete(ctx context.Context, id string) error HardDelete(ctx context.Context, id string) error + // HardDeleteFromAgent is HardDelete gated by the currently-assigned + // agent_name, for CE-event-driven callers that must reject a stale event + // from a superseded agent. Internal callers with no event to validate + // against (create rollback, non-deferred delete-on-agent-not-found) keep + // using HardDelete directly. + HardDeleteFromAgent(ctx context.Context, id string, agentName string) error + // MarkDeletionCompleteFromAgent is MarkDeletionComplete gated by the + // currently-assigned agent_name; see HardDeleteFromAgent. + MarkDeletionCompleteFromAgent(ctx context.Context, id string, agentName string) error ResetRetryCount(ctx context.Context, id string) error - MarkProviderDeletionsPendingProvider(ctx context.Context, providerName string) error - ReactivateProviderDeletions(ctx context.Context, providerName string) error - MarkPendingProviderIfNotReady(ctx context.Context, instanceID string) (bool, error) } type ServiceTypeInstanceStore struct { - db *gorm.DB - retryOpts []backoff.RetryOption + db *gorm.DB + retryOptsFunc func() []backoff.RetryOption } var _ ServiceTypeInstance = (*ServiceTypeInstanceStore)(nil) // NewServiceTypeInstance constructs the store. If no retry options are passed, -// production backoff is used for Create and HardDelete. Tests may pass custom -// options (e.g. sub-second intervals) to avoid slow retry exhaustion. +// production backoff is used for Create and HardDelete, built fresh on every +// call (retryOptsFunc) rather than shared across calls: *backoff.ExponentialBackOff +// carries mutable state (currentInterval) that Retry() mutates in place, so +// sharing one instance across concurrent Create/HardDelete calls would be a +// data race. Tests may pass custom options (e.g. sub-second intervals) to +// avoid slow retry exhaustion; those are reused as-is since tests don't +// exercise concurrent retry timing. func NewServiceTypeInstance(db *gorm.DB, retryOpts ...backoff.RetryOption) ServiceTypeInstance { - var opts []backoff.RetryOption if len(retryOpts) == 0 { - opts = getRetryOptions() - } else { - opts = append([]backoff.RetryOption(nil), retryOpts...) + return &ServiceTypeInstanceStore{db: db, retryOptsFunc: getRetryOptions} } - return &ServiceTypeInstanceStore{db: db, retryOpts: opts} + fixed := append([]backoff.RetryOption(nil), retryOpts...) + return &ServiceTypeInstanceStore{db: db, retryOptsFunc: func() []backoff.RetryOption { return fixed }} } func (s *ServiceTypeInstanceStore) List(ctx context.Context, opts *ServiceTypeInstanceListOptions) (*ServiceTypeInstanceListResult, error) { @@ -91,14 +111,14 @@ func (s *ServiceTypeInstanceStore) List(ctx context.Context, opts *ServiceTypeIn } // Apply filters - if opts != nil && opts.ProviderName != nil && *opts.ProviderName != "" { - query = query.Where("provider_name = ?", *opts.ProviderName) - } - if opts != nil && opts.ServiceType != nil && strings.TrimSpace(*opts.ServiceType) != "" { query = query.Where("service_type = ?", *opts.ServiceType) } + if opts != nil && opts.AgentName != nil && strings.TrimSpace(*opts.AgentName) != "" { + query = query.Where("agent_name = ?", *opts.AgentName) + } + // By default, exclude soft-deleted instances; show_deleted includes them if opts == nil || !opts.ShowDeleted { query = query.Where("deletion_status IS NULL") @@ -139,7 +159,7 @@ func (s *ServiceTypeInstanceStore) Create(ctx context.Context, instance model.Se return &instance, nil } - return backoff.Retry(ctx, operation, s.retryOpts...) + return backoff.Retry(ctx, operation, s.retryOptsFunc()...) } func (s *ServiceTypeInstanceStore) Get(ctx context.Context, id string, showDeleted bool) (*model.ServiceTypeInstance, error) { @@ -158,12 +178,16 @@ func (s *ServiceTypeInstanceStore) Get(ctx context.Context, id string, showDelet } func (s *ServiceTypeInstanceStore) UpdateStatus(ctx context.Context, instanceID string, status string, statusMessage string) error { + // Map-based Updates, not a struct literal: GORM's struct-based Updates + // skips zero-value fields, so a struct literal would silently ignore + // statusMessage == "" and leave a stale status_message (e.g. an old + // failure reason) attached to a status that no longer has one. result := s.db.WithContext(ctx). Model(&model.ServiceTypeInstance{}). Where("id = ?", instanceID). - Updates(model.ServiceTypeInstance{ - Status: status, - StatusMessage: statusMessage, + Updates(map[string]any{ + "status": status, + "status_message": statusMessage, }) if result.Error != nil { return result.Error @@ -174,6 +198,97 @@ func (s *ServiceTypeInstanceStore) UpdateStatus(ctx context.Context, instanceID return nil } +// UpdateStatusFrom atomically transitions status only if the instance's +// current status is one of fromStatuses AND its currently-assigned +// agent_name matches agentName, gated in the same WHERE clause (single +// atomic UPDATE) so a late event from an agent superseded by self-healing +// is rejected even if status cycled back into an allowed fromStatus under +// the new agent. Returns whether the update applied. +func (s *ServiceTypeInstanceStore) UpdateStatusFrom(ctx context.Context, instanceID string, fromStatuses []string, agentName string, status string, statusMessage string) (bool, error) { + result := s.db.WithContext(ctx). + Model(&model.ServiceTypeInstance{}). + Where("id = ? AND status IN ? AND agent_name = ?", instanceID, fromStatuses, agentName). + Updates(map[string]any{ + "status": status, + "status_message": statusMessage, + }) + if result.Error != nil { + return false, result.Error + } + return result.RowsAffected > 0, nil +} + +// MarkQueued transitions an instance to "queued" and resets pending_started_at +// so the queued-timeout sweep measures from the moment the agent queued the +// request. Gated on agent_name in the same WHERE clause, same as UpdateStatusFrom. +func (s *ServiceTypeInstanceStore) MarkQueued(ctx context.Context, id string, agentName string) error { + now := time.Now() + result := s.db.WithContext(ctx). + Model(&model.ServiceTypeInstance{}). + Where("id = ? AND status = ? AND agent_name = ?", id, model.StatusPending, agentName). + Updates(map[string]any{ + "status": model.StatusQueued, + "pending_started_at": now, + }) + if result.Error != nil { + return result.Error + } + if result.RowsAffected == 0 { + return ErrInstanceNotFound + } + return nil +} + +// ReassignAndReset re-points an instance at a new agent and resets it to a +// fresh "pending" state. retry_count is deliberately NOT reset: it's the +// cumulative count of self-heal attempts across every agent tried, so +// maxRetries is enforced globally even if a different agent is found each time. +// +// CAS-guarded to an ALLOW-list of exactly {pending, cancelled} - the two +// statuses the self-healing call sites call this from - rather than a +// deny-list of just {deleting, pending_deletion}: a deny-list would still +// let "provisioning" be reassigned, silently duplicating provisioning if the +// response consumer applies a creation-acknowledged in the same narrow +// window. Also requires deletion_status IS NULL, so an instance with a +// delete already scheduled can't be resurrected out from under the cleanup +// scheduler. Anything outside the allow-list returns ErrInstanceNotEligible. +// +// expectedCurrentAgent additionally CASes on agent_name: status alone stays +// "pending" across a successful reassignment, so two concurrent callers +// racing to reassign the SAME instance (e.g. two control-plane replicas, one +// via its own sweep claim and one via a sibling self-heal from a different +// resource in the same run) would otherwise both pass a status-only check +// and both publish a create to a different agent. Requiring the caller's +// observed agent_name to still match makes the second racer's update affect +// zero rows and fail closed with ErrInstanceNotEligible instead. +func (s *ServiceTypeInstanceStore) ReassignAndReset(ctx context.Context, id string, agentName string, expectedCurrentAgent string) error { + now := time.Now() + result := s.db.WithContext(ctx). + Model(&model.ServiceTypeInstance{}). + Where("id = ? AND status IN ? AND deletion_status IS NULL AND agent_name = ?", id, []string{model.StatusPending, model.StatusCancelled}, expectedCurrentAgent). + Updates(map[string]any{ + "agent_name": agentName, + "status": model.StatusPending, + "status_message": "", + "pending_started_at": now, + }) + if result.Error != nil { + return result.Error + } + if result.RowsAffected > 0 { + return nil + } + + exists, err := s.ExistsByID(ctx, id) + if err != nil { + return err + } + if !exists { + return ErrInstanceNotFound + } + return ErrInstanceNotEligible +} + func (s *ServiceTypeInstanceStore) ExistsByID(ctx context.Context, id string) (bool, error) { var instance model.ServiceTypeInstance err := s.db.WithContext(ctx).Select("id").Where("id = ?", id).Take(&instance).Error @@ -187,9 +302,8 @@ func (s *ServiceTypeInstanceStore) ExistsByID(ctx context.Context, id string) (b } const ( - DeletionStatusScheduled = "SCHEDULED" - DeletionStatusFailed = "FAILED" - DeletionStatusPendingProvider = "PENDING_PROVIDER" + DeletionStatusScheduled = "SCHEDULED" + DeletionStatusFailed = "FAILED" ) func (s *ServiceTypeInstanceStore) MarkForDeletion(ctx context.Context, id string) error { @@ -255,6 +369,41 @@ func (s *ServiceTypeInstanceStore) MarkDeletionFailed(ctx context.Context, id st return nil } +func (s *ServiceTypeInstanceStore) MarkDeletionComplete(ctx context.Context, id string) error { + result := s.db.WithContext(ctx). + Model(&model.ServiceTypeInstance{}). + Where("id = ?", id). + Update("deletion_status", "DELETED") + if result.Error != nil { + return result.Error + } + if result.RowsAffected == 0 { + return ErrInstanceNotFound + } + return nil +} + +// MarkDeletionCompleteFromAgent is MarkDeletionComplete additionally gated on +// agentName matching the instance's currently-assigned agent_name, for +// CE-event-driven callers (handleDeletionAcknowledged) that must not act on a +// stale event from a superseded agent. Returns ErrInstanceNotFound (matching +// MarkDeletionComplete's existing sentinel) both when the instance genuinely +// doesn't exist and when it exists but the agent doesn't match - callers +// already treat that sentinel as "ack, don't retry" either way. +func (s *ServiceTypeInstanceStore) MarkDeletionCompleteFromAgent(ctx context.Context, id string, agentName string) error { + result := s.db.WithContext(ctx). + Model(&model.ServiceTypeInstance{}). + Where("id = ? AND agent_name = ?", id, agentName). + Update("deletion_status", "DELETED") + if result.Error != nil { + return result.Error + } + if result.RowsAffected == 0 { + return ErrInstanceNotFound + } + return nil +} + func (s *ServiceTypeInstanceStore) HardDelete(ctx context.Context, id string) error { operation := func() (any, error) { result := s.db.WithContext(ctx).Unscoped().Where("id = ?", id).Delete(&model.ServiceTypeInstance{}) @@ -267,7 +416,27 @@ func (s *ServiceTypeInstanceStore) HardDelete(ctx context.Context, id string) er return nil, nil } - _, err := backoff.Retry(ctx, operation, s.retryOpts...) + _, err := backoff.Retry(ctx, operation, s.retryOptsFunc()...) + return err +} + +// HardDeleteFromAgent is HardDelete additionally gated on agentName matching +// the instance's currently-assigned agent_name; see +// MarkDeletionCompleteFromAgent for the rationale and error-sentinel +// behavior. +func (s *ServiceTypeInstanceStore) HardDeleteFromAgent(ctx context.Context, id string, agentName string) error { + operation := func() (any, error) { + result := s.db.WithContext(ctx).Unscoped().Where("id = ? AND agent_name = ?", id, agentName).Delete(&model.ServiceTypeInstance{}) + if result.Error != nil { + return nil, result.Error + } + if result.RowsAffected == 0 { + return nil, backoff.Permanent(ErrInstanceNotFound) + } + return nil, nil + } + + _, err := backoff.Retry(ctx, operation, s.retryOptsFunc()...) return err } @@ -289,39 +458,6 @@ func (s *ServiceTypeInstanceStore) ResetRetryCount(ctx context.Context, id strin return nil } -func (s *ServiceTypeInstanceStore) MarkProviderDeletionsPendingProvider(ctx context.Context, providerName string) error { - return s.db.WithContext(ctx). - Model(&model.ServiceTypeInstance{}). - Where("provider_name = ? AND deletion_status IN ?", providerName, []string{DeletionStatusScheduled, DeletionStatusFailed}). - Update("deletion_status", DeletionStatusPendingProvider). - Error -} - -func (s *ServiceTypeInstanceStore) ReactivateProviderDeletions(ctx context.Context, providerName string) error { - return s.db.WithContext(ctx). - Model(&model.ServiceTypeInstance{}). - Where("provider_name = ? AND deletion_status = ?", providerName, DeletionStatusPendingProvider). - Updates(map[string]any{ - "deletion_status": DeletionStatusScheduled, - "retry_count": 0, - }). - Error -} - -func (s *ServiceTypeInstanceStore) MarkPendingProviderIfNotReady(ctx context.Context, instanceID string) (bool, error) { - result := s.db.WithContext(ctx). - Model(&model.ServiceTypeInstance{}). - Where("id = ? AND provider_name IN (?)", - instanceID, - s.db.Model(&model.Provider{}).Select("name").Where("health_status IN ?", []model.HealthStatus{model.HealthStatusUnhealthy, model.HealthStatusUnavailable}), - ). - Update("deletion_status", DeletionStatusPendingProvider) - if result.Error != nil { - return false, result.Error - } - return result.RowsAffected > 0, nil -} - // getRetryOptions returns common retry configuration for database operations func getRetryOptions() []backoff.RetryOption { b := backoff.NewExponentialBackOff() diff --git a/internal/sp/store/resource_manager/service_instance_test.go b/internal/sp/store/resource_manager/service_instance_test.go index 26d1de1..a5fa76e 100644 --- a/internal/sp/store/resource_manager/service_instance_test.go +++ b/internal/sp/store/resource_manager/service_instance_test.go @@ -3,6 +3,7 @@ package store_test import ( "context" + agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" "github.com/dcm-project/control-plane/internal/sp/store/model" rmstore "github.com/dcm-project/control-plane/internal/sp/store/resource_manager" "github.com/dcm-project/control-plane/internal/sp/testutil" @@ -14,23 +15,26 @@ import ( "gorm.io/gorm/logger" ) -func newServiceTypeInstance(providerName, instanceName string, spec map[string]any) model.ServiceTypeInstance { +func newServiceTypeInstance(instanceName string, spec map[string]any) model.ServiceTypeInstance { return model.ServiceTypeInstance{ ID: uuid.New().String(), - ProviderName: providerName, Status: "PROVISIONING", InstanceName: instanceName, Spec: spec, } } -func newServiceTypeInstanceWithType(providerName, instanceName, serviceType string, spec map[string]any) model.ServiceTypeInstance { - inst := newServiceTypeInstance(providerName, instanceName, spec) +func newServiceTypeInstanceWithType(instanceName, serviceType string, spec map[string]any) model.ServiceTypeInstance { + inst := newServiceTypeInstance(instanceName, spec) inst.ServiceType = serviceType return inst } -var kubevirtProvider = "kubevirt-sp" +func newServiceTypeInstanceWithAgent(instanceName, agentName string, spec map[string]any) model.ServiceTypeInstance { + inst := newServiceTypeInstance(instanceName, spec) + inst.AgentName = &agentName + return inst +} var _ = Describe("ServiceTypeInstance Store", func() { var ( @@ -51,7 +55,7 @@ var _ = Describe("ServiceTypeInstance Store", func() { Logger: logger.Default.LogMode(logger.Silent), }) Expect(err).NotTo(HaveOccurred()) - Expect(db.AutoMigrate(&model.Provider{}, &model.ServiceTypeInstance{})).To(Succeed()) + Expect(db.AutoMigrate(&agentmodel.Agent{}, &model.ServiceTypeInstance{})).To(Succeed()) s = rmstore.NewServiceTypeInstance(db, testutil.FastServiceTypeInstanceRetry()...) ctx = context.Background() @@ -66,7 +70,6 @@ var _ = Describe("ServiceTypeInstance Store", func() { Describe("Create", func() { It("persists the instance", func() { instance := newServiceTypeInstance( - kubevirtProvider, "instance-1", map[string]any{"cpu": 2}) created, err := s.Create(ctx, instance) @@ -82,7 +85,6 @@ var _ = Describe("ServiceTypeInstance Store", func() { _ = sqlDB.Close() instance := newServiceTypeInstance( - kubevirtProvider, "retry-test", map[string]any{"cpu": 1}) @@ -96,14 +98,13 @@ var _ = Describe("ServiceTypeInstance Store", func() { Describe("Get", func() { It("retrieves by ID", func() { - seeded := newServiceTypeInstance(kubevirtProvider, "get-inst", map[string]any{"cpu": 1}) + seeded := newServiceTypeInstance("get-inst", map[string]any{"cpu": 1}) addInstanceToStore(seeded) found, err := s.Get(ctx, seeded.ID, false) Expect(err).NotTo(HaveOccurred()) Expect(found).NotTo(BeNil()) - Expect(found.ProviderName).To(Equal(kubevirtProvider)) Expect(found.InstanceName).To(Equal("get-inst")) }) @@ -115,9 +116,9 @@ var _ = Describe("ServiceTypeInstance Store", func() { Describe("List", func() { BeforeEach(func() { - addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "instance1", map[string]any{})) - addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "instance2", map[string]any{})) - addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "instance3", map[string]any{})) + addInstanceToStore(newServiceTypeInstance("instance1", map[string]any{})) + addInstanceToStore(newServiceTypeInstance("instance2", map[string]any{})) + addInstanceToStore(newServiceTypeInstance("instance3", map[string]any{})) }) It("returns all instances when opts is nil", func() { @@ -127,14 +128,6 @@ var _ = Describe("ServiceTypeInstance Store", func() { Expect(result.NextPageToken).To(BeNil()) }) - It("filters by provider name", func() { - result, err := s.List(ctx, &rmstore.ServiceTypeInstanceListOptions{ - ProviderName: &kubevirtProvider, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Instances).To(HaveLen(3)) - }) - It("applies pagination with page size", func() { result, err := s.List(ctx, &rmstore.ServiceTypeInstanceListOptions{ PageSize: 2, @@ -147,8 +140,8 @@ var _ = Describe("ServiceTypeInstance Store", func() { It("filters by service type", func() { vmType := "vm" containerType := "container" - addInstanceToStore(newServiceTypeInstanceWithType(kubevirtProvider, "vm-inst", vmType, map[string]any{})) - addInstanceToStore(newServiceTypeInstanceWithType(kubevirtProvider, "container-inst", containerType, map[string]any{})) + addInstanceToStore(newServiceTypeInstanceWithType("vm-inst", vmType, map[string]any{})) + addInstanceToStore(newServiceTypeInstanceWithType("container-inst", containerType, map[string]any{})) result, err := s.List(ctx, &rmstore.ServiceTypeInstanceListOptions{ ServiceType: &vmType, @@ -165,6 +158,65 @@ var _ = Describe("ServiceTypeInstance Store", func() { Expect(result.Instances[0].ServiceType).To(Equal("container")) }) + It("filters by agent name", func() { + addInstanceToStore(newServiceTypeInstanceWithAgent("agent-a-inst", "agent-a", map[string]any{})) + addInstanceToStore(newServiceTypeInstanceWithAgent("agent-b-inst", "agent-b", map[string]any{})) + + agentA := "agent-a" + result, err := s.List(ctx, &rmstore.ServiceTypeInstanceListOptions{ + AgentName: &agentA, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Instances).To(HaveLen(1)) + Expect(*result.Instances[0].AgentName).To(Equal("agent-a")) + }) + + It("excludes instances with no agent name when filtering by agent name", func() { + addInstanceToStore(newServiceTypeInstanceWithAgent("agent-a-inst", "agent-a", map[string]any{})) + addInstanceToStore(newServiceTypeInstance("unassigned-inst", map[string]any{})) + + agentA := "agent-a" + result, err := s.List(ctx, &rmstore.ServiceTypeInstanceListOptions{ + AgentName: &agentA, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Instances).To(HaveLen(1)) + Expect(*result.Instances[0].AgentName).To(Equal("agent-a")) + }) + + It("treats a blank agent name as no filter", func() { + addInstanceToStore(newServiceTypeInstanceWithAgent("agent-a-inst", "agent-a", map[string]any{})) + addInstanceToStore(newServiceTypeInstance("unassigned-inst", map[string]any{})) + + blank := " " + result, err := s.List(ctx, &rmstore.ServiceTypeInstanceListOptions{ + AgentName: &blank, + }) + Expect(err).NotTo(HaveOccurred()) + // 3 base instances from the outer BeforeEach + the 2 created above. + Expect(result.Instances).To(HaveLen(5)) + }) + + It("combines service type and agent name filters with AND semantics", func() { + vmType, containerType := "vm", "container" + agentA, agentB := "agent-a", "agent-b" + vmAgentA := newServiceTypeInstanceWithType("vm-agent-a", vmType, map[string]any{}) + vmAgentA.AgentName = &agentA + addInstanceToStore(vmAgentA) + vmAgentB := newServiceTypeInstanceWithType("vm-agent-b", vmType, map[string]any{}) + vmAgentB.AgentName = &agentB + addInstanceToStore(vmAgentB) + addInstanceToStore(newServiceTypeInstanceWithType("container-agent-a", containerType, map[string]any{})) + + result, err := s.List(ctx, &rmstore.ServiceTypeInstanceListOptions{ + ServiceType: &vmType, + AgentName: &agentA, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Instances).To(HaveLen(1)) + Expect(result.Instances[0].InstanceName).To(Equal("vm-agent-a")) + }) + It("returns next page using page token", func() { // Get first page firstPage, err := s.List(ctx, &rmstore.ServiceTypeInstanceListOptions{ @@ -187,7 +239,7 @@ var _ = Describe("ServiceTypeInstance Store", func() { Describe("HardDelete", func() { It("removes the instance", func() { - instance := newServiceTypeInstance(kubevirtProvider, "to-delete", map[string]any{}) + instance := newServiceTypeInstance("to-delete", map[string]any{}) addInstanceToStore(instance) Expect(s.HardDelete(ctx, instance.ID)).To(Succeed()) @@ -210,9 +262,30 @@ var _ = Describe("ServiceTypeInstance Store", func() { }) }) + // AC-08 (TC-11): the plain, unconditional variant used by internal + // (non-CE-driven) callers - no agent_name gating - must keep working + // exactly as before this fix, independent of the *FromAgent sibling. + Describe("MarkDeletionComplete", func() { + It("sets deletion_status to DELETED regardless of agent_name", func() { + instance := newServiceTypeInstance("soft-delete-plain", map[string]any{}) + addInstanceToStore(instance) + + Expect(s.MarkDeletionComplete(ctx, instance.ID)).To(Succeed()) + + found, err := s.Get(ctx, instance.ID, true) + Expect(err).NotTo(HaveOccurred()) + Expect(*found.DeletionStatus).To(Equal("DELETED")) + }) + + It("returns ErrInstanceNotFound for a genuinely missing ID", func() { + err := s.MarkDeletionComplete(ctx, uuid.New().String()) + Expect(err).To(MatchError(rmstore.ErrInstanceNotFound)) + }) + }) + Describe("UpdateStatus", func() { It("updates status and status message by instance ID", func() { - instance := newServiceTypeInstance(kubevirtProvider, "status-inst", map[string]any{"cpu": "2"}) + instance := newServiceTypeInstance("status-inst", map[string]any{"cpu": "2"}) addInstanceToStore(instance) err := s.UpdateStatus(ctx, instance.ID, "RUNNING", "VM is running") @@ -230,9 +303,325 @@ var _ = Describe("ServiceTypeInstance Store", func() { }) }) + Describe("UpdateStatusFrom", func() { + It("transitions status only when current status and agent_name match", func() { + agentName := "agent-a" + instance := newServiceTypeInstance("cas-inst", map[string]any{"cpu": "2"}) + instance.AgentName = &agentName + addInstanceToStore(instance) + + applied, err := s.UpdateStatusFrom(ctx, instance.ID, []string{"PROVISIONING"}, agentName, "RUNNING", "up") + Expect(err).NotTo(HaveOccurred()) + Expect(applied).To(BeTrue()) + + found, err := s.Get(ctx, instance.ID, false) + Expect(err).NotTo(HaveOccurred()) + Expect(found.Status).To(Equal("RUNNING")) + }) + + It("does not transition when current status does not match", func() { + agentName := "agent-a" + instance := newServiceTypeInstance("cas-inst-2", map[string]any{"cpu": "2"}) + instance.AgentName = &agentName + addInstanceToStore(instance) + + applied, err := s.UpdateStatusFrom(ctx, instance.ID, []string{"RUNNING"}, agentName, "STOPPED", "down") + Expect(err).NotTo(HaveOccurred()) + Expect(applied).To(BeFalse()) + + found, err := s.Get(ctx, instance.ID, false) + Expect(err).NotTo(HaveOccurred()) + Expect(found.Status).To(Equal("PROVISIONING")) + }) + + // An agent_name mismatch rejects an otherwise-valid transition, even + // though the status-only WHERE would have matched. + It("does not transition when status matches but agent_name does not (identity check)", func() { + currentAgent := "agent-b" + instance := newServiceTypeInstance("cas-inst-3", map[string]any{"cpu": "2"}) + instance.Status = "PENDING" + instance.AgentName = ¤tAgent + addInstanceToStore(instance) + + applied, err := s.UpdateStatusFrom(ctx, instance.ID, []string{"PENDING"}, "agent-a-stale", "PROVISIONING", "") + Expect(err).NotTo(HaveOccurred()) + Expect(applied).To(BeFalse()) + + found, err := s.Get(ctx, instance.ID, false) + Expect(err).NotTo(HaveOccurred()) + Expect(found.Status).To(Equal("PENDING")) + Expect(*found.AgentName).To(Equal(currentAgent)) + }) + + It("does not transition when the instance's agent_name is NULL (defensive: never treated as a match)", func() { + instance := newServiceTypeInstance("cas-inst-4", map[string]any{"cpu": "2"}) + instance.Status = "PENDING" + addInstanceToStore(instance) + + applied, err := s.UpdateStatusFrom(ctx, instance.ID, []string{"PENDING"}, "any-agent", "PROVISIONING", "") + Expect(err).NotTo(HaveOccurred()) + Expect(applied).To(BeFalse()) + }) + }) + + Describe("MarkQueued", func() { + It("transitions pending to queued and resets pending_started_at when agent_name matches", func() { + agentName := "agent-a" + instance := newServiceTypeInstance("queue-inst", map[string]any{}) + instance.Status = "pending" + instance.AgentName = &agentName + addInstanceToStore(instance) + + Expect(s.MarkQueued(ctx, instance.ID, agentName)).To(Succeed()) + + found, err := s.Get(ctx, instance.ID, false) + Expect(err).NotTo(HaveOccurred()) + Expect(found.Status).To(Equal("queued")) + Expect(found.PendingStartedAt).NotTo(BeNil()) + }) + + It("returns ErrInstanceNotFound when instance is not pending", func() { + agentName := "agent-a" + instance := newServiceTypeInstance("not-pending-inst", map[string]any{}) + instance.AgentName = &agentName + addInstanceToStore(instance) + + err := s.MarkQueued(ctx, instance.ID, agentName) + Expect(err).To(MatchError(rmstore.ErrInstanceNotFound)) + }) + + // An agent_name mismatch rejects an otherwise-valid pending -> + // queued transition, so a stale request-queued can't reset the + // queued timer under a new agent. + It("returns ErrInstanceNotFound when status matches but agent_name does not (identity check)", func() { + currentAgent := "agent-b" + instance := newServiceTypeInstance("queue-inst-mismatch", map[string]any{}) + instance.Status = "pending" + instance.AgentName = ¤tAgent + addInstanceToStore(instance) + + err := s.MarkQueued(ctx, instance.ID, "agent-a-stale") + Expect(err).To(MatchError(rmstore.ErrInstanceNotFound)) + + found, err := s.Get(ctx, instance.ID, false) + Expect(err).NotTo(HaveOccurred()) + Expect(found.Status).To(Equal("pending")) + }) + }) + + Describe("HardDeleteFromAgent", func() { + It("removes the instance when agent_name matches", func() { + agentName := "agent-a" + instance := newServiceTypeInstance("to-delete-agent", map[string]any{}) + instance.AgentName = &agentName + addInstanceToStore(instance) + + Expect(s.HardDeleteFromAgent(ctx, instance.ID, agentName)).To(Succeed()) + + _, err := s.Get(ctx, instance.ID, false) + Expect(err).To(MatchError(rmstore.ErrInstanceNotFound)) + }) + + // TC-12: mismatch is rejected with the same sentinel as "not found" - + // callers already treat that as "ack, don't retry" either way, and + // the row must survive (not be deleted) since the event's agent no + // longer owns it. + It("returns ErrInstanceNotFound and leaves the row intact when agent_name does not match", func() { + currentAgent := "agent-b" + instance := newServiceTypeInstance("to-delete-mismatch", map[string]any{}) + instance.AgentName = ¤tAgent + addInstanceToStore(instance) + + err := s.HardDeleteFromAgent(ctx, instance.ID, "agent-a-stale") + Expect(err).To(MatchError(rmstore.ErrInstanceNotFound)) + + found, getErr := s.Get(ctx, instance.ID, false) + Expect(getErr).NotTo(HaveOccurred()) + Expect(found.ID).To(Equal(instance.ID)) + }) + + It("returns ErrInstanceNotFound for a genuinely missing ID", func() { + err := s.HardDeleteFromAgent(ctx, uuid.New().String(), "agent-a") + Expect(err).To(MatchError(rmstore.ErrInstanceNotFound)) + }) + }) + + Describe("MarkDeletionCompleteFromAgent", func() { + It("sets deletion_status to DELETED when agent_name matches", func() { + agentName := "agent-a" + instance := newServiceTypeInstance("soft-delete-agent", map[string]any{}) + instance.AgentName = &agentName + addInstanceToStore(instance) + + Expect(s.MarkDeletionCompleteFromAgent(ctx, instance.ID, agentName)).To(Succeed()) + + found, err := s.Get(ctx, instance.ID, true) + Expect(err).NotTo(HaveOccurred()) + Expect(*found.DeletionStatus).To(Equal("DELETED")) + }) + + // TC-12: mismatch is rejected with the same sentinel as "not found", + // and does not touch deletion_status - the row must not be silently + // tombstoned by an event from an agent that no longer owns it. + It("returns ErrInstanceNotFound and leaves deletion_status untouched when agent_name does not match", func() { + currentAgent := "agent-b" + instance := newServiceTypeInstance("soft-delete-mismatch", map[string]any{}) + instance.AgentName = ¤tAgent + addInstanceToStore(instance) + + err := s.MarkDeletionCompleteFromAgent(ctx, instance.ID, "agent-a-stale") + Expect(err).To(MatchError(rmstore.ErrInstanceNotFound)) + + found, getErr := s.Get(ctx, instance.ID, true) + Expect(getErr).NotTo(HaveOccurred()) + Expect(found.DeletionStatus).To(BeNil()) + }) + + It("returns ErrInstanceNotFound for a genuinely missing ID", func() { + err := s.MarkDeletionCompleteFromAgent(ctx, uuid.New().String(), "agent-a") + Expect(err).To(MatchError(rmstore.ErrInstanceNotFound)) + }) + }) + + Describe("ReassignAndReset", func() { + It("re-points agent_name and resets to a fresh pending state, preserving retry_count", func() { + instance := newServiceTypeInstanceWithAgent("reassign-inst", "old-agent", map[string]any{}) + instance.Status = model.StatusPending + instance.RetryCount = 2 + addInstanceToStore(instance) + + Expect(s.ReassignAndReset(ctx, instance.ID, "new-agent", "old-agent")).To(Succeed()) + + found, err := s.Get(ctx, instance.ID, false) + Expect(err).NotTo(HaveOccurred()) + Expect(found.Status).To(Equal("pending")) + // retry_count is NOT reset: it's cumulative across every agent + // tried, so maxRetries is still enforced globally. + Expect(found.RetryCount).To(Equal(2)) + Expect(found.AgentName).NotTo(BeNil()) + Expect(*found.AgentName).To(Equal("new-agent")) + Expect(found.PendingStartedAt).NotTo(BeNil()) + }) + + It("returns ErrInstanceNotFound for missing ID", func() { + err := s.ReassignAndReset(ctx, uuid.New().String(), "new-agent", "old-agent") + Expect(err).To(MatchError(rmstore.ErrInstanceNotFound)) + }) + + It("returns ErrInstanceNotEligible when expectedCurrentAgent no longer matches (lost the race to a concurrent reassignment)", func() { + // Guards the fix for a real cross-replica race: two callers + // (e.g. a sweep-claimed primary heal and a sibling self-heal + // from a different resource in the same run) could otherwise + // both pass a status-only CAS and both publish a create to a + // different agent for the same instance, since status stays + // "pending" across a successful reassignment. + instance := newServiceTypeInstanceWithAgent("reassign-stale-agent", "agent-that-already-moved-on", map[string]any{}) + instance.Status = model.StatusPending + addInstanceToStore(instance) + + err := s.ReassignAndReset(ctx, instance.ID, "new-agent", "agent-the-caller-still-thinks-is-current") + Expect(err).To(MatchError(rmstore.ErrInstanceNotEligible)) + + found, getErr := s.Get(ctx, instance.ID, false) + Expect(getErr).NotTo(HaveOccurred()) + Expect(*found.AgentName).To(Equal("agent-that-already-moved-on")) + }) + + It("returns ErrInstanceNotEligible when the instance is mid-deletion", func() { + instance := newServiceTypeInstanceWithAgent("reassign-deleting", "old-agent", map[string]any{}) + instance.Status = model.StatusDeleting + addInstanceToStore(instance) + + err := s.ReassignAndReset(ctx, instance.ID, "new-agent", "old-agent") + Expect(err).To(MatchError(rmstore.ErrInstanceNotEligible)) + + found, getErr := s.Get(ctx, instance.ID, true) + Expect(getErr).NotTo(HaveOccurred()) + Expect(found.Status).To(Equal(model.StatusDeleting)) + }) + + It("returns ErrInstanceNotEligible for a pending_deletion instance", func() { + instance := newServiceTypeInstanceWithAgent("reassign-pending-deletion", "old-agent", map[string]any{}) + instance.Status = model.StatusPendingDeletion + addInstanceToStore(instance) + + err := s.ReassignAndReset(ctx, instance.ID, "new-agent", "old-agent") + Expect(err).To(MatchError(rmstore.ErrInstanceNotEligible)) + }) + + It("returns ErrInstanceNotEligible for a provisioning instance (split-brain: agent A may already be actively provisioning)", func() { + // Simulates: sweep claims a pending-timeout retry, then before + // selfHeal runs, the response consumer applies a genuine + // creation-acknowledged from the ORIGINAL agent (pending -> + // provisioning). ReassignAndReset must not silently re-point a + // "provisioning" instance onto a second agent - that would mean + // two agents both believe they own provisioning for the same + // instance. + originalAgent := "original-agent" + instance := newServiceTypeInstance("reassign-provisioning", map[string]any{}) + instance.Status = model.StatusProvisioning + instance.AgentName = &originalAgent + addInstanceToStore(instance) + + err := s.ReassignAndReset(ctx, instance.ID, "new-agent", originalAgent) + Expect(err).To(MatchError(rmstore.ErrInstanceNotEligible)) + + found, getErr := s.Get(ctx, instance.ID, true) + Expect(getErr).NotTo(HaveOccurred()) + Expect(found.Status).To(Equal(model.StatusProvisioning)) + Expect(found.AgentName).NotTo(BeNil()) + Expect(*found.AgentName).To(Equal(originalAgent)) + }) + + It("allows reassigning a cancelled instance (cancelQueuedInstance's self-heal path)", func() { + instance := newServiceTypeInstanceWithAgent("reassign-cancelled", "old-agent", map[string]any{}) + instance.Status = model.StatusCancelled + addInstanceToStore(instance) + + Expect(s.ReassignAndReset(ctx, instance.ID, "new-agent", "old-agent")).To(Succeed()) + + found, err := s.Get(ctx, instance.ID, false) + Expect(err).NotTo(HaveOccurred()) + Expect(found.Status).To(Equal(model.StatusPending)) + Expect(*found.AgentName).To(Equal("new-agent")) + }) + + It("returns ErrInstanceNotEligible for a queued instance (not yet cancelled by the sweep's CAS)", func() { + instance := newServiceTypeInstanceWithAgent("reassign-queued", "old-agent", map[string]any{}) + instance.Status = model.StatusQueued + addInstanceToStore(instance) + + err := s.ReassignAndReset(ctx, instance.ID, "new-agent", "old-agent") + Expect(err).To(MatchError(rmstore.ErrInstanceNotEligible)) + }) + + It("returns ErrInstanceNotEligible for a pending instance with a delete already scheduled against it (R2 S2: finding #1)", func() { + // A deferred DeleteInstance never touches Status, so a "pending" + // instance can have deletion_status=SCHEDULED. sweepPending's own + // filter is the primary defense, but this store method must not + // rely solely on its one caller to enforce that - it should + // refuse to resurrect/reassign such an instance no matter who + // calls it. + deletionStatus := "SCHEDULED" + instance := newServiceTypeInstanceWithAgent("reassign-delete-scheduled", "old-agent", map[string]any{}) + instance.Status = model.StatusPending + instance.DeletionStatus = &deletionStatus + addInstanceToStore(instance) + + err := s.ReassignAndReset(ctx, instance.ID, "new-agent", "old-agent") + Expect(err).To(MatchError(rmstore.ErrInstanceNotEligible)) + + found, getErr := s.Get(ctx, instance.ID, true) + Expect(getErr).NotTo(HaveOccurred()) + Expect(found.Status).To(Equal(model.StatusPending)) + Expect(found.DeletionStatus).NotTo(BeNil()) + Expect(*found.DeletionStatus).To(Equal("SCHEDULED")) + }) + }) + Describe("MarkForDeletion", func() { It("sets deletion_status to SCHEDULED", func() { - instance := newServiceTypeInstance(kubevirtProvider, "mark-del", map[string]any{}) + instance := newServiceTypeInstance("mark-del", map[string]any{}) addInstanceToStore(instance) Expect(s.MarkForDeletion(ctx, instance.ID)).To(Succeed()) @@ -245,7 +634,7 @@ var _ = Describe("ServiceTypeInstance Store", func() { }) It("hides instance from default Get", func() { - instance := newServiceTypeInstance(kubevirtProvider, "mark-hidden", map[string]any{}) + instance := newServiceTypeInstance("mark-hidden", map[string]any{}) addInstanceToStore(instance) Expect(s.MarkForDeletion(ctx, instance.ID)).To(Succeed()) @@ -262,9 +651,9 @@ var _ = Describe("ServiceTypeInstance Store", func() { Describe("ListPendingDeletions", func() { It("returns only SCHEDULED instances", func() { - inst1 := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "pending1", map[string]any{})) - inst2 := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "pending2", map[string]any{})) - addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "active", map[string]any{})) + inst1 := addInstanceToStore(newServiceTypeInstance("pending1", map[string]any{})) + inst2 := addInstanceToStore(newServiceTypeInstance("pending2", map[string]any{})) + addInstanceToStore(newServiceTypeInstance("active", map[string]any{})) Expect(s.MarkForDeletion(ctx, inst1.ID)).To(Succeed()) Expect(s.MarkForDeletion(ctx, inst2.ID)).To(Succeed()) @@ -275,7 +664,7 @@ var _ = Describe("ServiceTypeInstance Store", func() { }) It("excludes FAILED instances", func() { - inst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "failed", map[string]any{})) + inst := addInstanceToStore(newServiceTypeInstance("failed", map[string]any{})) Expect(s.MarkForDeletion(ctx, inst.ID)).To(Succeed()) Expect(s.MarkDeletionFailed(ctx, inst.ID)).To(Succeed()) @@ -285,7 +674,7 @@ var _ = Describe("ServiceTypeInstance Store", func() { }) It("returns empty when no pending deletions exist", func() { - addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "active", map[string]any{})) + addInstanceToStore(newServiceTypeInstance("active", map[string]any{})) pending, err := s.ListPendingDeletions(ctx) Expect(err).NotTo(HaveOccurred()) @@ -295,7 +684,7 @@ var _ = Describe("ServiceTypeInstance Store", func() { Describe("IncrementDeletionRetry", func() { It("increments retry count and sets last_deletion_attempt", func() { - inst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "retry-inst", map[string]any{})) + inst := addInstanceToStore(newServiceTypeInstance("retry-inst", map[string]any{})) Expect(s.MarkForDeletion(ctx, inst.ID)).To(Succeed()) Expect(s.IncrementDeletionRetry(ctx, inst.ID)).To(Succeed()) @@ -320,7 +709,7 @@ var _ = Describe("ServiceTypeInstance Store", func() { Describe("MarkDeletionFailed", func() { It("sets deletion_status to FAILED", func() { - inst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "fail-inst", map[string]any{})) + inst := addInstanceToStore(newServiceTypeInstance("fail-inst", map[string]any{})) Expect(s.MarkForDeletion(ctx, inst.ID)).To(Succeed()) Expect(s.MarkDeletionFailed(ctx, inst.ID)).To(Succeed()) @@ -338,7 +727,7 @@ var _ = Describe("ServiceTypeInstance Store", func() { Describe("ResetRetryCount", func() { It("resets retry count and status to SCHEDULED", func() { - inst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "reset-inst", map[string]any{})) + inst := addInstanceToStore(newServiceTypeInstance("reset-inst", map[string]any{})) Expect(s.MarkForDeletion(ctx, inst.ID)).To(Succeed()) Expect(s.IncrementDeletionRetry(ctx, inst.ID)).To(Succeed()) Expect(s.IncrementDeletionRetry(ctx, inst.ID)).To(Succeed()) @@ -361,7 +750,7 @@ var _ = Describe("ServiceTypeInstance Store", func() { Describe("Get with showDeleted", func() { It("returns soft-deleted instance when showDeleted is true", func() { - inst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "soft-del", map[string]any{})) + inst := addInstanceToStore(newServiceTypeInstance("soft-del", map[string]any{})) Expect(s.MarkForDeletion(ctx, inst.ID)).To(Succeed()) found, err := s.Get(ctx, inst.ID, true) @@ -371,7 +760,7 @@ var _ = Describe("ServiceTypeInstance Store", func() { }) It("returns not found for soft-deleted instance when showDeleted is false", func() { - inst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "soft-del2", map[string]any{})) + inst := addInstanceToStore(newServiceTypeInstance("soft-del2", map[string]any{})) Expect(s.MarkForDeletion(ctx, inst.ID)).To(Succeed()) _, err := s.Get(ctx, inst.ID, false) @@ -381,8 +770,8 @@ var _ = Describe("ServiceTypeInstance Store", func() { Describe("List with ShowDeleted", func() { It("excludes soft-deleted instances by default", func() { - addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "active", map[string]any{})) - deleted := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "deleted", map[string]any{})) + addInstanceToStore(newServiceTypeInstance("active", map[string]any{})) + deleted := addInstanceToStore(newServiceTypeInstance("deleted", map[string]any{})) Expect(s.MarkForDeletion(ctx, deleted.ID)).To(Succeed()) result, err := s.List(ctx, &rmstore.ServiceTypeInstanceListOptions{}) @@ -391,8 +780,8 @@ var _ = Describe("ServiceTypeInstance Store", func() { }) It("includes soft-deleted instances when ShowDeleted is true", func() { - addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "active", map[string]any{})) - deleted := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "deleted", map[string]any{})) + addInstanceToStore(newServiceTypeInstance("active", map[string]any{})) + deleted := addInstanceToStore(newServiceTypeInstance("deleted", map[string]any{})) Expect(s.MarkForDeletion(ctx, deleted.ID)).To(Succeed()) result, err := s.List(ctx, &rmstore.ServiceTypeInstanceListOptions{ShowDeleted: true}) @@ -403,7 +792,7 @@ var _ = Describe("ServiceTypeInstance Store", func() { Describe("ExistsByID", func() { It("returns true when instance exists", func() { - instance := newServiceTypeInstance(kubevirtProvider, "exists", map[string]any{}) + instance := newServiceTypeInstance("exists", map[string]any{}) addInstanceToStore(instance) exists, err := s.ExistsByID(ctx, instance.ID) @@ -417,156 +806,4 @@ var _ = Describe("ServiceTypeInstance Store", func() { Expect(exists).To(BeFalse()) }) }) - - Describe("MarkProviderDeletionsPendingProvider", func() { - It("transitions PENDING and FAILED instances to PENDING_PROVIDER", func() { - pendingInst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "pending", map[string]any{})) - Expect(s.MarkForDeletion(ctx, pendingInst.ID)).To(Succeed()) - - failedInst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "failed", map[string]any{})) - Expect(s.MarkForDeletion(ctx, failedInst.ID)).To(Succeed()) - Expect(s.MarkDeletionFailed(ctx, failedInst.ID)).To(Succeed()) - - activeInst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "active", map[string]any{})) - - Expect(s.MarkProviderDeletionsPendingProvider(ctx, kubevirtProvider)).To(Succeed()) - - found, err := s.Get(ctx, pendingInst.ID, true) - Expect(err).NotTo(HaveOccurred()) - Expect(*found.DeletionStatus).To(Equal("PENDING_PROVIDER")) - - found, err = s.Get(ctx, failedInst.ID, true) - Expect(err).NotTo(HaveOccurred()) - Expect(*found.DeletionStatus).To(Equal("PENDING_PROVIDER")) - - found, err = s.Get(ctx, activeInst.ID, false) - Expect(err).NotTo(HaveOccurred()) - Expect(found.DeletionStatus).To(BeNil()) - }) - - It("does not affect instances from other providers", func() { - otherProvider := "other-provider" - inst := addInstanceToStore(newServiceTypeInstance(otherProvider, "other-pending", map[string]any{})) - Expect(s.MarkForDeletion(ctx, inst.ID)).To(Succeed()) - - Expect(s.MarkProviderDeletionsPendingProvider(ctx, kubevirtProvider)).To(Succeed()) - - found, err := s.Get(ctx, inst.ID, true) - Expect(err).NotTo(HaveOccurred()) - Expect(*found.DeletionStatus).To(Equal("SCHEDULED")) - }) - }) - - Describe("ReactivateProviderDeletions", func() { - It("transitions PENDING_PROVIDER to PENDING with retry_count=0", func() { - inst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "marked", map[string]any{})) - Expect(s.MarkForDeletion(ctx, inst.ID)).To(Succeed()) - Expect(s.IncrementDeletionRetry(ctx, inst.ID)).To(Succeed()) - Expect(s.MarkProviderDeletionsPendingProvider(ctx, kubevirtProvider)).To(Succeed()) - - Expect(s.ReactivateProviderDeletions(ctx, kubevirtProvider)).To(Succeed()) - - found, err := s.Get(ctx, inst.ID, true) - Expect(err).NotTo(HaveOccurred()) - Expect(*found.DeletionStatus).To(Equal("SCHEDULED")) - Expect(found.RetryCount).To(Equal(0)) - }) - - It("does not affect SCHEDULED or FAILED instances", func() { - pendingInst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "still-pending", map[string]any{})) - Expect(s.MarkForDeletion(ctx, pendingInst.ID)).To(Succeed()) - - failedInst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "still-failed", map[string]any{})) - Expect(s.MarkForDeletion(ctx, failedInst.ID)).To(Succeed()) - Expect(s.MarkDeletionFailed(ctx, failedInst.ID)).To(Succeed()) - - Expect(s.ReactivateProviderDeletions(ctx, kubevirtProvider)).To(Succeed()) - - found, err := s.Get(ctx, pendingInst.ID, true) - Expect(err).NotTo(HaveOccurred()) - Expect(*found.DeletionStatus).To(Equal("SCHEDULED")) - - found, err = s.Get(ctx, failedInst.ID, true) - Expect(err).NotTo(HaveOccurred()) - Expect(*found.DeletionStatus).To(Equal("FAILED")) - }) - }) - - Describe("MarkPendingProviderIfNotReady", func() { - var addProvider func(name string, healthStatus model.HealthStatus) - - BeforeEach(func() { - addProvider = func(name string, healthStatus model.HealthStatus) { - provider := model.Provider{ - ID: uuid.New().String(), - Name: name, - ServiceType: "vm", - SchemaVersion: "v1", - Endpoint: "http://localhost:8080", - HealthStatus: healthStatus, - } - Expect(db.Create(&provider).Error).NotTo(HaveOccurred()) - } - }) - - It("parks instance and returns true when provider is Unavailable", func() { - addProvider(kubevirtProvider, model.HealthStatusUnavailable) - inst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "to-park", map[string]any{})) - Expect(s.MarkForDeletion(ctx, inst.ID)).To(Succeed()) - - marked, err := s.MarkPendingProviderIfNotReady(ctx, inst.ID) - Expect(err).NotTo(HaveOccurred()) - Expect(marked).To(BeTrue()) - - found, err := s.Get(ctx, inst.ID, true) - Expect(err).NotTo(HaveOccurred()) - Expect(*found.DeletionStatus).To(Equal("PENDING_PROVIDER")) - }) - - It("parks instance and returns true when provider is Unhealthy", func() { - addProvider(kubevirtProvider, model.HealthStatusUnhealthy) - inst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "to-park-unhealthy", map[string]any{})) - Expect(s.MarkForDeletion(ctx, inst.ID)).To(Succeed()) - - marked, err := s.MarkPendingProviderIfNotReady(ctx, inst.ID) - Expect(err).NotTo(HaveOccurred()) - Expect(marked).To(BeTrue()) - - found, err := s.Get(ctx, inst.ID, true) - Expect(err).NotTo(HaveOccurred()) - Expect(*found.DeletionStatus).To(Equal("PENDING_PROVIDER")) - }) - - It("is a no-op and returns false when provider is Ready", func() { - addProvider(kubevirtProvider, model.HealthStatusReady) - inst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "healthy", map[string]any{})) - Expect(s.MarkForDeletion(ctx, inst.ID)).To(Succeed()) - - marked, err := s.MarkPendingProviderIfNotReady(ctx, inst.ID) - Expect(err).NotTo(HaveOccurred()) - Expect(marked).To(BeFalse()) - - found, err := s.Get(ctx, inst.ID, true) - Expect(err).NotTo(HaveOccurred()) - Expect(*found.DeletionStatus).To(Equal("SCHEDULED")) - }) - - It("is idempotent when instance is already PENDING_PROVIDER", func() { - addProvider(kubevirtProvider, model.HealthStatusUnavailable) - inst := addInstanceToStore(newServiceTypeInstance(kubevirtProvider, "already-marked", map[string]any{})) - Expect(s.MarkForDeletion(ctx, inst.ID)).To(Succeed()) - - marked1, err := s.MarkPendingProviderIfNotReady(ctx, inst.ID) - Expect(err).NotTo(HaveOccurred()) - Expect(marked1).To(BeTrue()) - - marked2, err := s.MarkPendingProviderIfNotReady(ctx, inst.ID) - Expect(err).NotTo(HaveOccurred()) - Expect(marked2).To(BeTrue()) - - found, err := s.Get(ctx, inst.ID, true) - Expect(err).NotTo(HaveOccurred()) - Expect(*found.DeletionStatus).To(Equal("PENDING_PROVIDER")) - }) - }) }) diff --git a/internal/sp/store/store.go b/internal/sp/store/store.go index acfde07..9232281 100644 --- a/internal/sp/store/store.go +++ b/internal/sp/store/store.go @@ -3,20 +3,17 @@ package store import ( "github.com/cenkalti/backoff/v5" - providerstore "github.com/dcm-project/control-plane/internal/sp/store/provider" rmstore "github.com/dcm-project/control-plane/internal/sp/store/resource_manager" "gorm.io/gorm" ) type Store interface { Close() error - Provider() providerstore.Provider ServiceTypeInstance() rmstore.ServiceTypeInstance } type DataStore struct { db *gorm.DB - provider providerstore.Provider instance rmstore.ServiceTypeInstance } @@ -48,7 +45,6 @@ func NewStore(db *gorm.DB, opts ...StoreOption) Store { } return &DataStore{ db: db, - provider: providerstore.NewProvider(db), instance: instance, } } @@ -61,10 +57,6 @@ func (s *DataStore) Close() error { return sqlDB.Close() } -func (s *DataStore) Provider() providerstore.Provider { - return s.provider -} - func (s *DataStore) ServiceTypeInstance() rmstore.ServiceTypeInstance { return s.instance } diff --git a/internal/sp/store/store_test.go b/internal/sp/store/store_test.go index cd3291f..607d229 100644 --- a/internal/sp/store/store_test.go +++ b/internal/sp/store/store_test.go @@ -20,15 +20,6 @@ var _ = Describe("Store", func() { Expect(err).NotTo(HaveOccurred()) }) - Describe("NewStore", func() { - It("creates a store with provider access", func() { - s := store.NewStore(db) - - Expect(s).NotTo(BeNil()) - Expect(s.Provider()).NotTo(BeNil()) - }) - }) - Describe("NewStore", func() { It("creates a store with service type instance access", func() { s := store.NewStore(db) diff --git a/make/agent.mk b/make/agent.mk new file mode 100644 index 0000000..3cab4ca --- /dev/null +++ b/make/agent.mk @@ -0,0 +1,37 @@ +# Agent domain (codegen). +AGENT_DOMAIN := agent +AGENT_API := api/$(AGENT_DOMAIN)/v1alpha1 +AGENT_SERVER_DIR := internal/$(AGENT_DOMAIN)/api/server + +generate-agent-types: + go run github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen \ + --config=$(AGENT_API)/types.gen.cfg \ + -o $(AGENT_API)/types.gen.go \ + $(AGENT_API)/openapi.yaml + +generate-agent-spec: + go run github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen \ + --config=$(AGENT_API)/spec.gen.cfg \ + -o $(AGENT_API)/spec.gen.go \ + $(AGENT_API)/openapi.yaml + +generate-agent-server: + go run github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen \ + --config=$(AGENT_SERVER_DIR)/server.gen.cfg \ + -o $(AGENT_SERVER_DIR)/server.gen.go \ + $(AGENT_API)/openapi.yaml + +AGENT_CLIENT_DIR := pkg/$(AGENT_DOMAIN)/client + +generate-agent-client: + go run github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen \ + --config=$(AGENT_CLIENT_DIR)/client.gen.cfg \ + -o $(AGENT_CLIENT_DIR)/client.gen.go \ + $(AGENT_API)/openapi.yaml + +generate-agent-api: generate-agent-types generate-agent-spec generate-agent-server generate-agent-client + +test-agent: + $(GINKGO) $(GINKGO_FLAGS) ./internal/$(AGENT_DOMAIN)/... + +.PHONY: generate-agent-types generate-agent-spec generate-agent-server generate-agent-client generate-agent-api test-agent diff --git a/make/sp.mk b/make/sp.mk index 9389485..28b27fa 100644 --- a/make/sp.mk +++ b/make/sp.mk @@ -1,38 +1,9 @@ # Service provider domain (codegen and subsystem tests). SP_DOMAIN := sp -SP_PROVIDER_API := api/$(SP_DOMAIN)/v1alpha1/provider SP_RM_API := api/$(SP_DOMAIN)/v1alpha1/resource_manager -SP_PROVIDER_SERVER_DIR := internal/$(SP_DOMAIN)/api/provider SP_RM_SERVER_DIR := internal/$(SP_DOMAIN)/api/resource_manager -SP_PROVIDER_CLIENT_DIR := pkg/$(SP_DOMAIN)/client/provider SP_RM_CLIENT_DIR := pkg/$(SP_DOMAIN)/client/resource_manager -generate-sp-provider-types: - go run github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen \ - --config=$(SP_PROVIDER_API)/types.gen.cfg \ - -o $(SP_PROVIDER_API)/types.gen.go \ - $(SP_PROVIDER_API)/openapi.yaml - -generate-sp-provider-spec: - go run github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen \ - --config=$(SP_PROVIDER_API)/spec.gen.cfg \ - -o $(SP_PROVIDER_API)/spec.gen.go \ - $(SP_PROVIDER_API)/openapi.yaml - -generate-sp-provider-server: - go run github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen \ - --config=$(SP_PROVIDER_SERVER_DIR)/server.gen.cfg \ - -o $(SP_PROVIDER_SERVER_DIR)/server.gen.go \ - $(SP_PROVIDER_API)/openapi.yaml - -generate-sp-provider-client: - go run github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen \ - --config=$(SP_PROVIDER_CLIENT_DIR)/client.gen.cfg \ - -o $(SP_PROVIDER_CLIENT_DIR)/client.gen.go \ - $(SP_PROVIDER_API)/openapi.yaml - -generate-sp-provider-api: generate-sp-provider-types generate-sp-provider-spec generate-sp-provider-server generate-sp-provider-client - generate-sp-rm-types: go run github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen \ --config=$(SP_RM_API)/types.gen.cfg \ @@ -59,15 +30,12 @@ generate-sp-rm-client: generate-sp-rm-api: generate-sp-rm-types generate-sp-rm-spec generate-sp-rm-server generate-sp-rm-client -generate-sp-api: generate-sp-provider-api generate-sp-rm-api - -check-sp-aep-provider: - spectral lint --fail-severity=warn ./$(SP_PROVIDER_API)/openapi.yaml +generate-sp-api: generate-sp-rm-api check-sp-aep-rm: spectral lint --fail-severity=warn ./$(SP_RM_API)/openapi.yaml -check-sp-aep: check-sp-aep-provider check-sp-aep-rm +check-sp-aep: check-sp-aep-rm test-sp: $(GINKGO) $(GINKGO_FLAGS) ./internal/$(SP_DOMAIN) @@ -81,8 +49,7 @@ sp-subsystem-test-down: sp-subsystem-test: $(GINKGO) $(GINKGO_FLAGS) -tags=subsystem ./test/subsystem/$(SP_DOMAIN) -.PHONY: generate-sp-provider-types generate-sp-provider-spec generate-sp-provider-server \ - generate-sp-provider-client generate-sp-provider-api generate-sp-rm-types generate-sp-rm-spec \ +.PHONY: generate-sp-rm-types generate-sp-rm-spec \ generate-sp-rm-server generate-sp-rm-client generate-sp-rm-api generate-sp-api \ - check-sp-aep-provider check-sp-aep-rm check-sp-aep test-sp \ + check-sp-aep-rm check-sp-aep test-sp \ sp-subsystem-test-up sp-subsystem-test-down sp-subsystem-test diff --git a/pkg/agent/client/client.gen.cfg b/pkg/agent/client/client.gen.cfg new file mode 100644 index 0000000..108d503 --- /dev/null +++ b/pkg/agent/client/client.gen.cfg @@ -0,0 +1,8 @@ +package: client +generate: + client: true +additional-imports: + - alias: . + package: github.com/dcm-project/control-plane/api/agent/v1alpha1 +output-options: + skip-prune: true diff --git a/pkg/agent/client/client.gen.go b/pkg/agent/client/client.gen.go new file mode 100644 index 0000000..959888b --- /dev/null +++ b/pkg/agent/client/client.gen.go @@ -0,0 +1,811 @@ +// Package client provides primitives to interact with the openapi HTTP API. +// +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.7.0 DO NOT EDIT. +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + . "github.com/dcm-project/control-plane/api/agent/v1alpha1" + "github.com/oapi-codegen/runtime" +) + +// RequestEditorFn is the function signature for the RequestEditor callback function +type RequestEditorFn func(ctx context.Context, req *http.Request) error + +// Doer performs HTTP requests. +// +// The standard http.Client implements this interface. +type HttpRequestDoer interface { + Do(req *http.Request) (*http.Response, error) +} + +// Client which conforms to the OpenAPI3 specification for this service. +type Client struct { + // The endpoint of the server conforming to this interface, with scheme, + // https://api.deepmap.com for example. This can contain a path relative + // to the server, such as https://api.deepmap.com/dev-test, and all the + // paths in the swagger spec will be appended to the server. + Server string + + // Doer for performing requests, typically a *http.Client with any + // customized settings, such as certificate chains. + Client HttpRequestDoer + + // A list of callbacks for modifying requests which are generated before sending over + // the network. + RequestEditors []RequestEditorFn +} + +// ClientOption allows setting custom parameters during construction +type ClientOption func(*Client) error + +// Creates a new Client, with reasonable defaults +func NewClient(server string, opts ...ClientOption) (*Client, error) { + // create a client with sane default values + client := Client{ + Server: server, + } + // mutate client and add all optional params + for _, o := range opts { + if err := o(&client); err != nil { + return nil, err + } + } + // ensure the server URL always has a trailing slash + if !strings.HasSuffix(client.Server, "/") { + client.Server += "/" + } + // create httpClient, if not already present + if client.Client == nil { + client.Client = &http.Client{} + } + return &client, nil +} + +// WithHTTPClient allows overriding the default Doer, which is +// automatically created using http.Client. This is useful for tests. +func WithHTTPClient(doer HttpRequestDoer) ClientOption { + return func(c *Client) error { + c.Client = doer + return nil + } +} + +// WithRequestEditorFn allows setting up a callback function, which will be +// called right before sending the request. This can be used to mutate the request. +func WithRequestEditorFn(fn RequestEditorFn) ClientOption { + return func(c *Client) error { + c.RequestEditors = append(c.RequestEditors, fn) + return nil + } +} + +// The interface specification for the client above. +type ClientInterface interface { + // ListAgents request + ListAgents(ctx context.Context, params *ListAgentsParams, reqEditors ...RequestEditorFn) (*http.Response, error) + + // CreateAgentWithBody request with any body + CreateAgentWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + CreateAgent(ctx context.Context, body CreateAgentJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + + // GetAgent request + GetAgent(ctx context.Context, agentId AgentIdPath, reqEditors ...RequestEditorFn) (*http.Response, error) + + // AgentHeartbeatWithBody request with any body + AgentHeartbeatWithBody(ctx context.Context, agentId AgentIdPath, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + AgentHeartbeat(ctx context.Context, agentId AgentIdPath, body AgentHeartbeatJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) +} + +func (c *Client) ListAgents(ctx context.Context, params *ListAgentsParams, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewListAgentsRequest(c.Server, params) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) CreateAgentWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewCreateAgentRequestWithBody(c.Server, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) CreateAgent(ctx context.Context, body CreateAgentJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewCreateAgentRequest(c.Server, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) GetAgent(ctx context.Context, agentId AgentIdPath, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetAgentRequest(c.Server, agentId) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) AgentHeartbeatWithBody(ctx context.Context, agentId AgentIdPath, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewAgentHeartbeatRequestWithBody(c.Server, agentId, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) AgentHeartbeat(ctx context.Context, agentId AgentIdPath, body AgentHeartbeatJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewAgentHeartbeatRequest(c.Server, agentId, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +// NewListAgentsRequest generates requests for ListAgents +func NewListAgentsRequest(server string, params *ListAgentsParams) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/agents") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + if params != nil { + // queryValues collects non-styled parameters (passthrough, JSON) + // that are safe to round-trip through url.Values.Encode(). + queryValues := queryURL.Query() + // rawQueryFragments collects pre-encoded query fragments from + // styled parameters, preserving literal commas as delimiters + // per the OpenAPI spec (e.g. "color=blue,black,brown"). + var rawQueryFragments []string + + if params.HealthStatus != nil { + + if queryFrag, err := runtime.StyleParamWithOptions("form", true, "health_status", *params.HealthStatus, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationQuery, Type: "string", Format: ""}); err != nil { + return nil, err + } else { + for _, qp := range strings.Split(queryFrag, "&") { + rawQueryFragments = append(rawQueryFragments, qp) + } + } + + } + + if params.MaxPageSize != nil { + + if queryFrag, err := runtime.StyleParamWithOptions("form", true, "max_page_size", *params.MaxPageSize, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationQuery, Type: "integer", Format: ""}); err != nil { + return nil, err + } else { + for _, qp := range strings.Split(queryFrag, "&") { + rawQueryFragments = append(rawQueryFragments, qp) + } + } + + } + + if params.PageToken != nil { + + if queryFrag, err := runtime.StyleParamWithOptions("form", true, "page_token", *params.PageToken, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationQuery, Type: "string", Format: ""}); err != nil { + return nil, err + } else { + for _, qp := range strings.Split(queryFrag, "&") { + rawQueryFragments = append(rawQueryFragments, qp) + } + } + + } + + if encoded := queryValues.Encode(); encoded != "" { + rawQueryFragments = append(rawQueryFragments, encoded) + } + queryURL.RawQuery = strings.Join(rawQueryFragments, "&") + } + + req, err := http.NewRequest(http.MethodGet, queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewCreateAgentRequest calls the generic CreateAgent builder with application/json body +func NewCreateAgentRequest(server string, body CreateAgentJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewCreateAgentRequestWithBody(server, "application/json", bodyReader) +} + +// NewCreateAgentRequestWithBody generates requests for CreateAgent with any type of body +func NewCreateAgentRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/agents") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPost, queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + +// NewGetAgentRequest generates requests for GetAgent +func NewGetAgentRequest(server string, agentId AgentIdPath) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithOptions("simple", false, "agentId", agentId, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/agents/%s", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodGet, queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewAgentHeartbeatRequest calls the generic AgentHeartbeat builder with application/json body +func NewAgentHeartbeatRequest(server string, agentId AgentIdPath, body AgentHeartbeatJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewAgentHeartbeatRequestWithBody(server, agentId, "application/json", bodyReader) +} + +// NewAgentHeartbeatRequestWithBody generates requests for AgentHeartbeat with any type of body +func NewAgentHeartbeatRequestWithBody(server string, agentId AgentIdPath, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithOptions("simple", false, "agentId", agentId, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/agents/%s/heartbeat", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPut, queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + +func (c *Client) applyEditors(ctx context.Context, req *http.Request, additionalEditors []RequestEditorFn) error { + for _, r := range c.RequestEditors { + if err := r(ctx, req); err != nil { + return err + } + } + for _, r := range additionalEditors { + if err := r(ctx, req); err != nil { + return err + } + } + return nil +} + +// ClientWithResponses builds on ClientInterface to offer response payloads +type ClientWithResponses struct { + ClientInterface +} + +// NewClientWithResponses creates a new ClientWithResponses, which wraps +// Client with return type handling +func NewClientWithResponses(server string, opts ...ClientOption) (*ClientWithResponses, error) { + client, err := NewClient(server, opts...) + if err != nil { + return nil, err + } + return &ClientWithResponses{client}, nil +} + +// WithBaseURL overrides the baseURL. +func WithBaseURL(baseURL string) ClientOption { + return func(c *Client) error { + newBaseURL, err := url.Parse(baseURL) + if err != nil { + return err + } + c.Server = newBaseURL.String() + return nil + } +} + +// ClientWithResponsesInterface is the interface specification for the client with responses above. +type ClientWithResponsesInterface interface { + // ListAgentsWithResponse request + ListAgentsWithResponse(ctx context.Context, params *ListAgentsParams, reqEditors ...RequestEditorFn) (*ListAgentsResponse, error) + + // CreateAgentWithBodyWithResponse request with any body + CreateAgentWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateAgentResponse, error) + + CreateAgentWithResponse(ctx context.Context, body CreateAgentJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateAgentResponse, error) + + // GetAgentWithResponse request + GetAgentWithResponse(ctx context.Context, agentId AgentIdPath, reqEditors ...RequestEditorFn) (*GetAgentResponse, error) + + // AgentHeartbeatWithBodyWithResponse request with any body + AgentHeartbeatWithBodyWithResponse(ctx context.Context, agentId AgentIdPath, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*AgentHeartbeatResponse, error) + + AgentHeartbeatWithResponse(ctx context.Context, agentId AgentIdPath, body AgentHeartbeatJSONRequestBody, reqEditors ...RequestEditorFn) (*AgentHeartbeatResponse, error) +} + +type ListAgentsResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *AgentList + ApplicationproblemJSON400 *Error + ApplicationproblemJSONDefault *Error +} + +// Status returns HTTPResponse.Status +func (r ListAgentsResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r ListAgentsResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r ListAgentsResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + +type CreateAgentResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *Agent + JSON201 *Agent + ApplicationproblemJSON400 *Error + ApplicationproblemJSON409 *Error + ApplicationproblemJSONDefault *Error +} + +// Status returns HTTPResponse.Status +func (r CreateAgentResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r CreateAgentResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r CreateAgentResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + +type GetAgentResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *Agent + ApplicationproblemJSON400 *Error + ApplicationproblemJSON404 *Error + ApplicationproblemJSONDefault *Error +} + +// Status returns HTTPResponse.Status +func (r GetAgentResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetAgentResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r GetAgentResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + +type AgentHeartbeatResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *Agent + ApplicationproblemJSON400 *Error + ApplicationproblemJSON404 *Error + ApplicationproblemJSONDefault *Error +} + +// Status returns HTTPResponse.Status +func (r AgentHeartbeatResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r AgentHeartbeatResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r AgentHeartbeatResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + +// ListAgentsWithResponse request returning *ListAgentsResponse +func (c *ClientWithResponses) ListAgentsWithResponse(ctx context.Context, params *ListAgentsParams, reqEditors ...RequestEditorFn) (*ListAgentsResponse, error) { + rsp, err := c.ListAgents(ctx, params, reqEditors...) + if err != nil { + return nil, err + } + return ParseListAgentsResponse(rsp) +} + +// CreateAgentWithBodyWithResponse request with arbitrary body returning *CreateAgentResponse +func (c *ClientWithResponses) CreateAgentWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateAgentResponse, error) { + rsp, err := c.CreateAgentWithBody(ctx, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseCreateAgentResponse(rsp) +} + +func (c *ClientWithResponses) CreateAgentWithResponse(ctx context.Context, body CreateAgentJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateAgentResponse, error) { + rsp, err := c.CreateAgent(ctx, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseCreateAgentResponse(rsp) +} + +// GetAgentWithResponse request returning *GetAgentResponse +func (c *ClientWithResponses) GetAgentWithResponse(ctx context.Context, agentId AgentIdPath, reqEditors ...RequestEditorFn) (*GetAgentResponse, error) { + rsp, err := c.GetAgent(ctx, agentId, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetAgentResponse(rsp) +} + +// AgentHeartbeatWithBodyWithResponse request with arbitrary body returning *AgentHeartbeatResponse +func (c *ClientWithResponses) AgentHeartbeatWithBodyWithResponse(ctx context.Context, agentId AgentIdPath, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*AgentHeartbeatResponse, error) { + rsp, err := c.AgentHeartbeatWithBody(ctx, agentId, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseAgentHeartbeatResponse(rsp) +} + +func (c *ClientWithResponses) AgentHeartbeatWithResponse(ctx context.Context, agentId AgentIdPath, body AgentHeartbeatJSONRequestBody, reqEditors ...RequestEditorFn) (*AgentHeartbeatResponse, error) { + rsp, err := c.AgentHeartbeat(ctx, agentId, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseAgentHeartbeatResponse(rsp) +} + +// ParseListAgentsResponse parses an HTTP response from a ListAgentsWithResponse call +func ParseListAgentsResponse(rsp *http.Response) (*ListAgentsResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &ListAgentsResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest AgentList + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSONDefault = &dest + + } + + return response, nil +} + +// ParseCreateAgentResponse parses an HTTP response from a CreateAgentWithResponse call +func ParseCreateAgentResponse(rsp *http.Response) (*CreateAgentResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &CreateAgentResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest Agent + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 201: + var dest Agent + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON201 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 409: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON409 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSONDefault = &dest + + } + + return response, nil +} + +// ParseGetAgentResponse parses an HTTP response from a GetAgentWithResponse call +func ParseGetAgentResponse(rsp *http.Response) (*GetAgentResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetAgentResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest Agent + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSONDefault = &dest + + } + + return response, nil +} + +// ParseAgentHeartbeatResponse parses an HTTP response from a AgentHeartbeatWithResponse call +func ParseAgentHeartbeatResponse(rsp *http.Response) (*AgentHeartbeatResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &AgentHeartbeatResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest Agent + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSONDefault = &dest + + } + + return response, nil +} diff --git a/pkg/sp/client/provider/client.gen.cfg b/pkg/sp/client/provider/client.gen.cfg deleted file mode 100644 index a0fb482..0000000 --- a/pkg/sp/client/provider/client.gen.cfg +++ /dev/null @@ -1,9 +0,0 @@ -package: provider -generate: - client: true -additional-imports: - - alias: . - package: github.com/dcm-project/control-plane/api/sp/v1alpha1/provider -output-options: - skip-prune: true - diff --git a/pkg/sp/client/provider/client.gen.go b/pkg/sp/client/provider/client.gen.go deleted file mode 100644 index 5a9e3bc..0000000 --- a/pkg/sp/client/provider/client.gen.go +++ /dev/null @@ -1,1067 +0,0 @@ -// Package provider provides primitives to interact with the openapi HTTP API. -// -// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.7.0 DO NOT EDIT. -package provider - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - - . "github.com/dcm-project/control-plane/api/sp/v1alpha1/provider" - "github.com/oapi-codegen/runtime" -) - -// RequestEditorFn is the function signature for the RequestEditor callback function -type RequestEditorFn func(ctx context.Context, req *http.Request) error - -// Doer performs HTTP requests. -// -// The standard http.Client implements this interface. -type HttpRequestDoer interface { - Do(req *http.Request) (*http.Response, error) -} - -// Client which conforms to the OpenAPI3 specification for this service. -type Client struct { - // The endpoint of the server conforming to this interface, with scheme, - // https://api.deepmap.com for example. This can contain a path relative - // to the server, such as https://api.deepmap.com/dev-test, and all the - // paths in the swagger spec will be appended to the server. - Server string - - // Doer for performing requests, typically a *http.Client with any - // customized settings, such as certificate chains. - Client HttpRequestDoer - - // A list of callbacks for modifying requests which are generated before sending over - // the network. - RequestEditors []RequestEditorFn -} - -// ClientOption allows setting custom parameters during construction -type ClientOption func(*Client) error - -// Creates a new Client, with reasonable defaults -func NewClient(server string, opts ...ClientOption) (*Client, error) { - // create a client with sane default values - client := Client{ - Server: server, - } - // mutate client and add all optional params - for _, o := range opts { - if err := o(&client); err != nil { - return nil, err - } - } - // ensure the server URL always has a trailing slash - if !strings.HasSuffix(client.Server, "/") { - client.Server += "/" - } - // create httpClient, if not already present - if client.Client == nil { - client.Client = &http.Client{} - } - return &client, nil -} - -// WithHTTPClient allows overriding the default Doer, which is -// automatically created using http.Client. This is useful for tests. -func WithHTTPClient(doer HttpRequestDoer) ClientOption { - return func(c *Client) error { - c.Client = doer - return nil - } -} - -// WithRequestEditorFn allows setting up a callback function, which will be -// called right before sending the request. This can be used to mutate the request. -func WithRequestEditorFn(fn RequestEditorFn) ClientOption { - return func(c *Client) error { - c.RequestEditors = append(c.RequestEditors, fn) - return nil - } -} - -// The interface specification for the client above. -type ClientInterface interface { - // ListProviders request - ListProviders(ctx context.Context, params *ListProvidersParams, reqEditors ...RequestEditorFn) (*http.Response, error) - - // CreateProviderWithBody request with any body - CreateProviderWithBody(ctx context.Context, params *CreateProviderParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) - - CreateProvider(ctx context.Context, params *CreateProviderParams, body CreateProviderJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - - // DeleteProvider request - DeleteProvider(ctx context.Context, providerId ProviderIdPath, reqEditors ...RequestEditorFn) (*http.Response, error) - - // GetProvider request - GetProvider(ctx context.Context, providerId ProviderIdPath, reqEditors ...RequestEditorFn) (*http.Response, error) - - // ApplyProviderWithBody request with any body - ApplyProviderWithBody(ctx context.Context, providerId ProviderIdPath, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) - - ApplyProvider(ctx context.Context, providerId ProviderIdPath, body ApplyProviderJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) -} - -func (c *Client) ListProviders(ctx context.Context, params *ListProvidersParams, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewListProvidersRequest(c.Server, params) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) CreateProviderWithBody(ctx context.Context, params *CreateProviderParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewCreateProviderRequestWithBody(c.Server, params, contentType, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) CreateProvider(ctx context.Context, params *CreateProviderParams, body CreateProviderJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewCreateProviderRequest(c.Server, params, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) DeleteProvider(ctx context.Context, providerId ProviderIdPath, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewDeleteProviderRequest(c.Server, providerId) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) GetProvider(ctx context.Context, providerId ProviderIdPath, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewGetProviderRequest(c.Server, providerId) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) ApplyProviderWithBody(ctx context.Context, providerId ProviderIdPath, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewApplyProviderRequestWithBody(c.Server, providerId, contentType, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) ApplyProvider(ctx context.Context, providerId ProviderIdPath, body ApplyProviderJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewApplyProviderRequest(c.Server, providerId, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -// NewListProvidersRequest generates requests for ListProviders -func NewListProvidersRequest(server string, params *ListProvidersParams) (*http.Request, error) { - var err error - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/providers") - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - if params != nil { - // queryValues collects non-styled parameters (passthrough, JSON) - // that are safe to round-trip through url.Values.Encode(). - queryValues := queryURL.Query() - // rawQueryFragments collects pre-encoded query fragments from - // styled parameters, preserving literal commas as delimiters - // per the OpenAPI spec (e.g. "color=blue,black,brown"). - var rawQueryFragments []string - - if params.Type != nil { - - if queryFrag, err := runtime.StyleParamWithOptions("form", true, "type", *params.Type, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationQuery, Type: "string", Format: ""}); err != nil { - return nil, err - } else { - for _, qp := range strings.Split(queryFrag, "&") { - rawQueryFragments = append(rawQueryFragments, qp) - } - } - - } - - if params.MaxPageSize != nil { - - if queryFrag, err := runtime.StyleParamWithOptions("form", true, "max_page_size", *params.MaxPageSize, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationQuery, Type: "integer", Format: ""}); err != nil { - return nil, err - } else { - for _, qp := range strings.Split(queryFrag, "&") { - rawQueryFragments = append(rawQueryFragments, qp) - } - } - - } - - if params.PageToken != nil { - - if queryFrag, err := runtime.StyleParamWithOptions("form", true, "page_token", *params.PageToken, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationQuery, Type: "string", Format: ""}); err != nil { - return nil, err - } else { - for _, qp := range strings.Split(queryFrag, "&") { - rawQueryFragments = append(rawQueryFragments, qp) - } - } - - } - - if encoded := queryValues.Encode(); encoded != "" { - rawQueryFragments = append(rawQueryFragments, encoded) - } - queryURL.RawQuery = strings.Join(rawQueryFragments, "&") - } - - req, err := http.NewRequest(http.MethodGet, queryURL.String(), nil) - if err != nil { - return nil, err - } - - return req, nil -} - -// NewCreateProviderRequest calls the generic CreateProvider builder with application/json body -func NewCreateProviderRequest(server string, params *CreateProviderParams, body CreateProviderJSONRequestBody) (*http.Request, error) { - var bodyReader io.Reader - buf, err := json.Marshal(body) - if err != nil { - return nil, err - } - bodyReader = bytes.NewReader(buf) - return NewCreateProviderRequestWithBody(server, params, "application/json", bodyReader) -} - -// NewCreateProviderRequestWithBody generates requests for CreateProvider with any type of body -func NewCreateProviderRequestWithBody(server string, params *CreateProviderParams, contentType string, body io.Reader) (*http.Request, error) { - var err error - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/providers") - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - if params != nil { - // queryValues collects non-styled parameters (passthrough, JSON) - // that are safe to round-trip through url.Values.Encode(). - queryValues := queryURL.Query() - // rawQueryFragments collects pre-encoded query fragments from - // styled parameters, preserving literal commas as delimiters - // per the OpenAPI spec (e.g. "color=blue,black,brown"). - var rawQueryFragments []string - - if params.Id != nil { - - if queryFrag, err := runtime.StyleParamWithOptions("form", true, "id", *params.Id, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationQuery, Type: "string", Format: ""}); err != nil { - return nil, err - } else { - for _, qp := range strings.Split(queryFrag, "&") { - rawQueryFragments = append(rawQueryFragments, qp) - } - } - - } - - if encoded := queryValues.Encode(); encoded != "" { - rawQueryFragments = append(rawQueryFragments, encoded) - } - queryURL.RawQuery = strings.Join(rawQueryFragments, "&") - } - - req, err := http.NewRequest(http.MethodPost, queryURL.String(), body) - if err != nil { - return nil, err - } - - req.Header.Add("Content-Type", contentType) - - return req, nil -} - -// NewDeleteProviderRequest generates requests for DeleteProvider -func NewDeleteProviderRequest(server string, providerId ProviderIdPath) (*http.Request, error) { - var err error - - var pathParam0 string - - pathParam0, err = runtime.StyleParamWithOptions("simple", false, "providerId", providerId, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) - if err != nil { - return nil, err - } - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/providers/%s", pathParam0) - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(http.MethodDelete, queryURL.String(), nil) - if err != nil { - return nil, err - } - - return req, nil -} - -// NewGetProviderRequest generates requests for GetProvider -func NewGetProviderRequest(server string, providerId ProviderIdPath) (*http.Request, error) { - var err error - - var pathParam0 string - - pathParam0, err = runtime.StyleParamWithOptions("simple", false, "providerId", providerId, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) - if err != nil { - return nil, err - } - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/providers/%s", pathParam0) - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(http.MethodGet, queryURL.String(), nil) - if err != nil { - return nil, err - } - - return req, nil -} - -// NewApplyProviderRequest calls the generic ApplyProvider builder with application/json body -func NewApplyProviderRequest(server string, providerId ProviderIdPath, body ApplyProviderJSONRequestBody) (*http.Request, error) { - var bodyReader io.Reader - buf, err := json.Marshal(body) - if err != nil { - return nil, err - } - bodyReader = bytes.NewReader(buf) - return NewApplyProviderRequestWithBody(server, providerId, "application/json", bodyReader) -} - -// NewApplyProviderRequestWithBody generates requests for ApplyProvider with any type of body -func NewApplyProviderRequestWithBody(server string, providerId ProviderIdPath, contentType string, body io.Reader) (*http.Request, error) { - var err error - - var pathParam0 string - - pathParam0, err = runtime.StyleParamWithOptions("simple", false, "providerId", providerId, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) - if err != nil { - return nil, err - } - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/providers/%s", pathParam0) - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - req, err := http.NewRequest(http.MethodPut, queryURL.String(), body) - if err != nil { - return nil, err - } - - req.Header.Add("Content-Type", contentType) - - return req, nil -} - -func (c *Client) applyEditors(ctx context.Context, req *http.Request, additionalEditors []RequestEditorFn) error { - for _, r := range c.RequestEditors { - if err := r(ctx, req); err != nil { - return err - } - } - for _, r := range additionalEditors { - if err := r(ctx, req); err != nil { - return err - } - } - return nil -} - -// ClientWithResponses builds on ClientInterface to offer response payloads -type ClientWithResponses struct { - ClientInterface -} - -// NewClientWithResponses creates a new ClientWithResponses, which wraps -// Client with return type handling -func NewClientWithResponses(server string, opts ...ClientOption) (*ClientWithResponses, error) { - client, err := NewClient(server, opts...) - if err != nil { - return nil, err - } - return &ClientWithResponses{client}, nil -} - -// WithBaseURL overrides the baseURL. -func WithBaseURL(baseURL string) ClientOption { - return func(c *Client) error { - newBaseURL, err := url.Parse(baseURL) - if err != nil { - return err - } - c.Server = newBaseURL.String() - return nil - } -} - -// ClientWithResponsesInterface is the interface specification for the client with responses above. -type ClientWithResponsesInterface interface { - // ListProvidersWithResponse request - ListProvidersWithResponse(ctx context.Context, params *ListProvidersParams, reqEditors ...RequestEditorFn) (*ListProvidersResponse, error) - - // CreateProviderWithBodyWithResponse request with any body - CreateProviderWithBodyWithResponse(ctx context.Context, params *CreateProviderParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateProviderResponse, error) - - CreateProviderWithResponse(ctx context.Context, params *CreateProviderParams, body CreateProviderJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateProviderResponse, error) - - // DeleteProviderWithResponse request - DeleteProviderWithResponse(ctx context.Context, providerId ProviderIdPath, reqEditors ...RequestEditorFn) (*DeleteProviderResponse, error) - - // GetProviderWithResponse request - GetProviderWithResponse(ctx context.Context, providerId ProviderIdPath, reqEditors ...RequestEditorFn) (*GetProviderResponse, error) - - // ApplyProviderWithBodyWithResponse request with any body - ApplyProviderWithBodyWithResponse(ctx context.Context, providerId ProviderIdPath, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*ApplyProviderResponse, error) - - ApplyProviderWithResponse(ctx context.Context, providerId ProviderIdPath, body ApplyProviderJSONRequestBody, reqEditors ...RequestEditorFn) (*ApplyProviderResponse, error) -} - -type ListProvidersResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *ProviderList - ApplicationproblemJSON400 *Error - JSON401 *Unauthorized - JSON403 *Forbidden - ApplicationproblemJSONDefault *Error -} - -// Status returns HTTPResponse.Status -func (r ListProvidersResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r ListProvidersResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - -// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers -func (r ListProvidersResponse) ContentType() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Header.Get("Content-Type") - } - return "" -} - -type CreateProviderResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *Provider - JSON201 *Provider - ApplicationproblemJSON400 *Error - JSON401 *Unauthorized - JSON403 *Forbidden - ApplicationproblemJSON409 *Error - ApplicationproblemJSON422 *Error - ApplicationproblemJSONDefault *Error -} - -// Status returns HTTPResponse.Status -func (r CreateProviderResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r CreateProviderResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - -// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers -func (r CreateProviderResponse) ContentType() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Header.Get("Content-Type") - } - return "" -} - -type DeleteProviderResponse struct { - Body []byte - HTTPResponse *http.Response - ApplicationproblemJSON400 *Error - JSON401 *Unauthorized - JSON403 *Forbidden - ApplicationproblemJSON404 *Error - ApplicationproblemJSONDefault *Error -} - -// Status returns HTTPResponse.Status -func (r DeleteProviderResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r DeleteProviderResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - -// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers -func (r DeleteProviderResponse) ContentType() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Header.Get("Content-Type") - } - return "" -} - -type GetProviderResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *Provider - ApplicationproblemJSON400 *Error - JSON401 *Unauthorized - JSON403 *Forbidden - ApplicationproblemJSON404 *Error - ApplicationproblemJSONDefault *Error -} - -// Status returns HTTPResponse.Status -func (r GetProviderResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r GetProviderResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - -// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers -func (r GetProviderResponse) ContentType() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Header.Get("Content-Type") - } - return "" -} - -type ApplyProviderResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *Provider - ApplicationproblemJSON400 *Error - JSON401 *Unauthorized - JSON403 *Forbidden - ApplicationproblemJSON404 *Error - ApplicationproblemJSON409 *Error - ApplicationproblemJSONDefault *Error -} - -// Status returns HTTPResponse.Status -func (r ApplyProviderResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r ApplyProviderResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - -// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers -func (r ApplyProviderResponse) ContentType() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Header.Get("Content-Type") - } - return "" -} - -// ListProvidersWithResponse request returning *ListProvidersResponse -func (c *ClientWithResponses) ListProvidersWithResponse(ctx context.Context, params *ListProvidersParams, reqEditors ...RequestEditorFn) (*ListProvidersResponse, error) { - rsp, err := c.ListProviders(ctx, params, reqEditors...) - if err != nil { - return nil, err - } - return ParseListProvidersResponse(rsp) -} - -// CreateProviderWithBodyWithResponse request with arbitrary body returning *CreateProviderResponse -func (c *ClientWithResponses) CreateProviderWithBodyWithResponse(ctx context.Context, params *CreateProviderParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateProviderResponse, error) { - rsp, err := c.CreateProviderWithBody(ctx, params, contentType, body, reqEditors...) - if err != nil { - return nil, err - } - return ParseCreateProviderResponse(rsp) -} - -func (c *ClientWithResponses) CreateProviderWithResponse(ctx context.Context, params *CreateProviderParams, body CreateProviderJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateProviderResponse, error) { - rsp, err := c.CreateProvider(ctx, params, body, reqEditors...) - if err != nil { - return nil, err - } - return ParseCreateProviderResponse(rsp) -} - -// DeleteProviderWithResponse request returning *DeleteProviderResponse -func (c *ClientWithResponses) DeleteProviderWithResponse(ctx context.Context, providerId ProviderIdPath, reqEditors ...RequestEditorFn) (*DeleteProviderResponse, error) { - rsp, err := c.DeleteProvider(ctx, providerId, reqEditors...) - if err != nil { - return nil, err - } - return ParseDeleteProviderResponse(rsp) -} - -// GetProviderWithResponse request returning *GetProviderResponse -func (c *ClientWithResponses) GetProviderWithResponse(ctx context.Context, providerId ProviderIdPath, reqEditors ...RequestEditorFn) (*GetProviderResponse, error) { - rsp, err := c.GetProvider(ctx, providerId, reqEditors...) - if err != nil { - return nil, err - } - return ParseGetProviderResponse(rsp) -} - -// ApplyProviderWithBodyWithResponse request with arbitrary body returning *ApplyProviderResponse -func (c *ClientWithResponses) ApplyProviderWithBodyWithResponse(ctx context.Context, providerId ProviderIdPath, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*ApplyProviderResponse, error) { - rsp, err := c.ApplyProviderWithBody(ctx, providerId, contentType, body, reqEditors...) - if err != nil { - return nil, err - } - return ParseApplyProviderResponse(rsp) -} - -func (c *ClientWithResponses) ApplyProviderWithResponse(ctx context.Context, providerId ProviderIdPath, body ApplyProviderJSONRequestBody, reqEditors ...RequestEditorFn) (*ApplyProviderResponse, error) { - rsp, err := c.ApplyProvider(ctx, providerId, body, reqEditors...) - if err != nil { - return nil, err - } - return ParseApplyProviderResponse(rsp) -} - -// ParseListProvidersResponse parses an HTTP response from a ListProvidersWithResponse call -func ParseListProvidersResponse(rsp *http.Response) (*ListProvidersResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &ListProvidersResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest ProviderList - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSON400 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: - var dest Unauthorized - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON401 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403: - var dest Forbidden - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON403 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSONDefault = &dest - - } - - return response, nil -} - -// ParseCreateProviderResponse parses an HTTP response from a CreateProviderWithResponse call -func ParseCreateProviderResponse(rsp *http.Response) (*CreateProviderResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &CreateProviderResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest Provider - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 201: - var dest Provider - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON201 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSON400 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: - var dest Unauthorized - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON401 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403: - var dest Forbidden - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON403 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 409: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSON409 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 422: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSON422 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSONDefault = &dest - - } - - return response, nil -} - -// ParseDeleteProviderResponse parses an HTTP response from a DeleteProviderWithResponse call -func ParseDeleteProviderResponse(rsp *http.Response) (*DeleteProviderResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &DeleteProviderResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSON400 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: - var dest Unauthorized - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON401 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403: - var dest Forbidden - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON403 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSON404 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSONDefault = &dest - - } - - return response, nil -} - -// ParseGetProviderResponse parses an HTTP response from a GetProviderWithResponse call -func ParseGetProviderResponse(rsp *http.Response) (*GetProviderResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &GetProviderResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest Provider - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSON400 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: - var dest Unauthorized - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON401 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403: - var dest Forbidden - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON403 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSON404 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSONDefault = &dest - - } - - return response, nil -} - -// ParseApplyProviderResponse parses an HTTP response from a ApplyProviderWithResponse call -func ParseApplyProviderResponse(rsp *http.Response) (*ApplyProviderResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &ApplyProviderResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest Provider - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSON400 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: - var dest Unauthorized - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON401 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403: - var dest Forbidden - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON403 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSON404 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 409: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSON409 = &dest - - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: - var dest Error - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.ApplicationproblemJSONDefault = &dest - - } - - return response, nil -} diff --git a/pkg/sp/client/resource_manager/client.gen.go b/pkg/sp/client/resource_manager/client.gen.go index 159b539..8eff5bc 100644 --- a/pkg/sp/client/resource_manager/client.gen.go +++ b/pkg/sp/client/resource_manager/client.gen.go @@ -193,9 +193,9 @@ func NewListInstancesRequest(server string, params *ListInstancesParams) (*http. // per the OpenAPI spec (e.g. "color=blue,black,brown"). var rawQueryFragments []string - if params.Provider != nil { + if params.ServiceType != nil { - if queryFrag, err := runtime.StyleParamWithOptions("form", true, "provider", *params.Provider, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationQuery, Type: "string", Format: ""}); err != nil { + if queryFrag, err := runtime.StyleParamWithOptions("form", true, "service_type", *params.ServiceType, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationQuery, Type: "string", Format: ""}); err != nil { return nil, err } else { for _, qp := range strings.Split(queryFrag, "&") { @@ -205,9 +205,9 @@ func NewListInstancesRequest(server string, params *ListInstancesParams) (*http. } - if params.ServiceType != nil { + if params.AgentName != nil { - if queryFrag, err := runtime.StyleParamWithOptions("form", true, "service_type", *params.ServiceType, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationQuery, Type: "string", Format: ""}); err != nil { + if queryFrag, err := runtime.StyleParamWithOptions("form", true, "agent_name", *params.AgentName, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationQuery, Type: "string", Format: ""}); err != nil { return nil, err } else { for _, qp := range strings.Split(queryFrag, "&") { @@ -558,6 +558,7 @@ type CreateInstanceResponse struct { ApplicationproblemJSON404 *Error ApplicationproblemJSON409 *Error ApplicationproblemJSON422 *Error + ApplicationproblemJSON503 *Error ApplicationproblemJSONDefault *Error } @@ -592,6 +593,7 @@ type DeleteInstanceResponse struct { JSON401 *Unauthorized JSON403 *Forbidden ApplicationproblemJSON404 *Error + ApplicationproblemJSON422 *Error ApplicationproblemJSONDefault *Error } @@ -815,6 +817,13 @@ func ParseCreateInstanceResponse(rsp *http.Response) (*CreateInstanceResponse, e } response.ApplicationproblemJSON422 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 503: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON503 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { @@ -869,6 +878,13 @@ func ParseDeleteInstanceResponse(rsp *http.Response) (*DeleteInstanceResponse, e } response.ApplicationproblemJSON404 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 422: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.ApplicationproblemJSON422 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && true: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { diff --git a/test/subsystem/sp/docker-compose.yaml b/test/subsystem/sp/docker-compose.yaml index 50fcc17..ac9081d 100644 --- a/test/subsystem/sp/docker-compose.yaml +++ b/test/subsystem/sp/docker-compose.yaml @@ -57,6 +57,16 @@ services: NATS_URL: "nats://nats:4222" AUTH_DISABLED: "true" DCM_ADMIN_SUBJECT: "test-admin-sub" + # Tuned down from the 2m/10s production defaults so the self-heal + # subsystem test doesn't have to wait minutes for a real sweep cycle. + AGENT_PENDING_REQUEST_TIMEOUT: "5s" + AGENT_SWEEP_INTERVAL: "2s" + # Tuned down from the default of 3 so the retries-exhausted test + # (single agent, no fallback) reaches "failed" in ~10-15s instead of + # ~25-30s. Verified this doesn't change the outcome of the existing + # two-agent self-heal tests: they resolve on their first retry + # attempt, well under either limit. + AGENT_PENDING_REQUEST_MAX_RETRIES: "1" ports: - "8080:8080" depends_on: diff --git a/test/subsystem/sp/provider_test.go b/test/subsystem/sp/provider_test.go deleted file mode 100644 index f50f68c..0000000 --- a/test/subsystem/sp/provider_test.go +++ /dev/null @@ -1,151 +0,0 @@ -//go:build subsystem - -package subsystem_test - -import ( - "context" - "net/http" - "os" - - providerapi "github.com/dcm-project/control-plane/api/sp/v1alpha1/provider" - providerclient "github.com/dcm-project/control-plane/pkg/sp/client/provider" - "github.com/google/uuid" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Provider API", func() { - var ( - apiClient *providerclient.ClientWithResponses - ctx context.Context - ) - - BeforeEach(func() { - baseURL := os.Getenv("API_URL") - if baseURL == "" { - baseURL = "http://localhost:8080/api/v1alpha1" - } - - var err error - authEditor := providerclient.WithRequestEditorFn(func(_ context.Context, req *http.Request) error { - req.Header.Set("X-Forwarded-User", "test-admin-sub") - return nil - }) - apiClient, err = providerclient.NewClientWithResponses(baseURL, authEditor) - Expect(err).NotTo(HaveOccurred()) - - ctx = context.Background() - }) - - Describe("Health", func() { - It("returns healthy status", func() { - baseURL := os.Getenv("API_URL") - if baseURL == "" { - baseURL = "http://localhost:8080/api/v1alpha1" - } - - resp, err := http.Get(baseURL + "/health") - Expect(err).NotTo(HaveOccurred()) - defer resp.Body.Close() - Expect(resp.StatusCode).To(Equal(http.StatusOK)) - }) - }) - - Describe("Provider CRUD", func() { - It("creates, reads, updates, and deletes a provider", func() { - By("creating a new provider") - createResp, err := apiClient.CreateProviderWithResponse(ctx, nil, providerapi.Provider{ - Name: "e2e-test-provider", - Endpoint: "https://example.com/api", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(createResp.StatusCode()).To(Equal(http.StatusCreated)) - Expect(createResp.JSON201).NotTo(BeNil()) - Expect(createResp.JSON201.Id).NotTo(BeNil()) - - providerID := *createResp.JSON201.Id - - By("getting the provider") - getResp, err := apiClient.GetProviderWithResponse(ctx, providerID) - Expect(err).NotTo(HaveOccurred()) - Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) - Expect(getResp.JSON200.Name).To(Equal("e2e-test-provider")) - - By("re-registering without ID (idempotent update)") - reregResp, err := apiClient.CreateProviderWithResponse(ctx, nil, providerapi.Provider{ - Name: "e2e-test-provider", - Endpoint: "https://updated.example.com/api", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(reregResp.StatusCode()).To(Equal(http.StatusOK)) - Expect(reregResp.JSON200).NotTo(BeNil()) - Expect(*reregResp.JSON200.Id).To(Equal(providerID)) - - By("listing providers") - listResp, err := apiClient.ListProvidersWithResponse(ctx, nil) - Expect(err).NotTo(HaveOccurred()) - Expect(listResp.StatusCode()).To(Equal(http.StatusOK)) - Expect(listResp.JSON200.Providers).NotTo(BeNil()) - Expect(len(*listResp.JSON200.Providers)).To(BeNumerically(">=", 1)) - - By("updating the provider") - updateResp, err := apiClient.ApplyProviderWithResponse(ctx, providerID, providerapi.Provider{ - Name: "e2e-test-provider-updated", - Endpoint: "https://updated.example.com/api", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(updateResp.StatusCode()).To(Equal(http.StatusOK)) - Expect(updateResp.JSON200.Name).To(Equal("e2e-test-provider-updated")) - - By("deleting the provider") - deleteResp, err := apiClient.DeleteProviderWithResponse(ctx, providerID) - Expect(err).NotTo(HaveOccurred()) - Expect(deleteResp.StatusCode()).To(Equal(http.StatusNoContent)) - - By("verifying provider is deleted") - getDeletedResp, err := apiClient.GetProviderWithResponse(ctx, providerID) - Expect(err).NotTo(HaveOccurred()) - Expect(getDeletedResp.StatusCode()).To(Equal(http.StatusNotFound)) - }) - }) - - Describe("Conflict scenarios", func() { - var providerID string - - BeforeEach(func() { - resp, err := apiClient.CreateProviderWithResponse(ctx, nil, providerapi.Provider{ - Name: "conflict-test-provider", - Endpoint: "https://example.com/api", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.StatusCode()).To(Equal(http.StatusCreated)) - providerID = *resp.JSON201.Id - }) - - AfterEach(func() { - apiClient.DeleteProviderWithResponse(ctx, providerID) - }) - - It("returns 409 when registering same name with different ID", func() { - newID := uuid.New().String() - params := &providerapi.CreateProviderParams{Id: &newID} - - resp, err := apiClient.CreateProviderWithResponse(ctx, params, providerapi.Provider{ - Name: "conflict-test-provider", - Endpoint: "https://other.example.com/api", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(resp.StatusCode()).To(Equal(http.StatusConflict)) - }) - }) -}) diff --git a/test/subsystem/sp/response_events_test.go b/test/subsystem/sp/response_events_test.go new file mode 100644 index 0000000..ce06ecb --- /dev/null +++ b/test/subsystem/sp/response_events_test.go @@ -0,0 +1,95 @@ +//go:build subsystem + +package subsystem_test + +import ( + "context" + "net/http" + "time" + + "github.com/dcm-project/control-plane/api/sp/v1alpha1/resource_manager" + "github.com/dcm-project/control-plane/internal/sp/messaging" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// eventuallyStatus polls the instance's status via the API against a real +// NATS/JetStream broker: the response consumer processes the published +// event asynchronously, so the transition isn't guaranteed to be visible +// immediately after publish. ShowDeleted is always set: pending_deletion is +// one of the statuses this helper polls for, and GetInstance 404s on +// soft-deleted instances otherwise. +// +// Assertions run through the polled func(g Gomega) rather than the package- +// level Expect, so a single transient failure (e.g. a momentary non-200) +// is treated as "try again next poll" instead of aborting the spec outright. +func eventuallyStatus(instanceID string) AsyncAssertion { + return Eventually(func(g Gomega) string { + params := &resource_manager.GetInstanceParams{ShowDeleted: ptr(true)} + getResp, err := rmApiClient.GetInstanceWithResponse(context.Background(), instanceID, params) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) + g.Expect(getResp.JSON200.Status).NotTo(BeNil()) + return *getResp.JSON200.Status + }).WithTimeout(10 * time.Second).WithPolling(500 * time.Millisecond) +} + +var _ = Describe("Agent response events", func() { + BeforeEach(func() { + resetWireMock() + }) + + It("moves a pending instance to provisioning on creation-acknowledged", func() { + agentName, instanceID := createInstanceViaAgent() + + publishResponseEvent(messaging.CETypeCreationAcknowledged, instanceID, agentName) + + eventuallyStatus(instanceID).Should(Equal("provisioning")) + }) + + It("moves a pending instance to queued on request-queued", func() { + agentName, instanceID := createInstanceViaAgent() + + publishResponseEvent(messaging.CETypeRequestQueued, instanceID, agentName) + + eventuallyStatus(instanceID).Should(Equal("queued")) + }) + + It("moves an instance to failed on error", func() { + agentName, instanceID := createInstanceViaAgent() + + publishResponseEvent(messaging.CETypeError, instanceID, agentName) + + eventuallyStatus(instanceID).Should(Equal("failed")) + }) + + It("moves a queued instance to cancelled on cancel-acknowledged", func() { + agentName, instanceID := createInstanceViaAgent() + publishResponseEvent(messaging.CETypeRequestQueued, instanceID, agentName) + eventuallyStatus(instanceID).Should(Equal("queued")) + + publishResponseEvent(messaging.CETypeCancelAcknowledged, instanceID, agentName) + + eventuallyStatus(instanceID).Should(Equal("cancelled")) + }) + + It("moves a queued instance to pending_deletion on cancel-rejected", func() { + agentName, instanceID := createInstanceViaAgent() + publishResponseEvent(messaging.CETypeRequestQueued, instanceID, agentName) + eventuallyStatus(instanceID).Should(Equal("queued")) + + publishResponseEvent(messaging.CETypeCancelRejected, instanceID, agentName) + + eventuallyStatus(instanceID).Should(Equal("pending_deletion")) + + // The instanceStatus/eventuallyStatus helpers above pass + // ShowDeleted:true because pending_deletion is soft-deleted; verify + // that's actually load-bearing, not just a harmless default, by + // confirming the plain (default-filtered) GET eventually 404s. + Eventually(func(g Gomega) int { + getResp, err := rmApiClient.GetInstanceWithResponse(context.Background(), instanceID, nil) + g.Expect(err).NotTo(HaveOccurred()) + return getResp.StatusCode() + }).WithTimeout(5 * time.Second).WithPolling(100 * time.Millisecond).Should(Equal(http.StatusNotFound)) + }) +}) diff --git a/test/subsystem/sp/self_heal_test.go b/test/subsystem/sp/self_heal_test.go new file mode 100644 index 0000000..3b51a5c --- /dev/null +++ b/test/subsystem/sp/self_heal_test.go @@ -0,0 +1,295 @@ +//go:build subsystem + +package subsystem_test + +import ( + "context" + "net/http" + "time" + + catalogapi "github.com/dcm-project/control-plane/api/catalog/v1alpha1" + "github.com/dcm-project/control-plane/internal/sp/messaging" + "github.com/google/uuid" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// createCatalogItemInstance creates a catalog item instance from +// catalogItemID and registers its own cleanup. Shared by every self-heal +// test below (which don't otherwise need the resulting instance/agent +// identity createInstanceViaAgent returns, since they look resources up via +// findInstanceByAgentName/listInstanceIDsByAgent/ +// findInstanceByAgentAndServiceType instead). +func createCatalogItemInstance(catalogItemID string) { + instID := "sp-subsystem-inst-" + uuid.New().String()[:8] + instParams := &catalogapi.CreateCatalogItemInstanceParams{Id: &instID} + instBody := catalogapi.CatalogItemInstance{ + ApiVersion: "v1alpha1", + DisplayName: instID, // display_name is capped at 63 chars by the schema + Spec: catalogapi.CatalogItemInstanceSpec{ + CatalogItemId: catalogItemID, + UserValues: []catalogapi.UserValue{}, + }, + } + instResp, err := catalogApiClient.CreateCatalogItemInstanceWithResponse(context.Background(), instParams, instBody) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, instResp.StatusCode()).To(Equal(http.StatusCreated)) + + DeferCleanup(func() { + _, _ = catalogApiClient.DeleteCatalogItemInstanceWithResponse(context.Background(), instID) + }) +} + +// Self-heal loop: a pending instance whose agent never acknowledges it gets +// automatically re-routed to an alternate agent once AGENT_PENDING_REQUEST_TIMEOUT +// elapses, driven by the real sweep against the real NATS/JetStream broker +// (docker-compose.yaml tunes both timeouts down for this test). +var _ = Describe("Self-heal on pending timeout", func() { + BeforeEach(func() { + resetWireMock() + }) + + It("reassigns a never-acknowledged pending instance to the alternate ready agent", func() { + const serviceType = "vm" + + agentA := registerReadyAgent(serviceType) + agentB := registerReadyAgent(serviceType) + policyID := createTwoAgentPolicy(agentA, agentB) + DeferCleanup(func() { + _, _ = policyApiClient.DeletePolicyWithResponse(context.Background(), policyID) + }) + + catalogItemID := createTestCatalogItem(serviceType) + createCatalogItemInstance(catalogItemID) + + // agentA/agentB are freshly minted per-test names: querying by + // agent_name directly (rather than scanning a fixed-size page of + // ListInstances, oldest-first) finds the instance regardless of how + // many instances the shared suite-wide DB has accumulated. + instanceID, initialAgent := findInstanceByEitherAgent(agentA, agentB) + + expectedAlternate := agentB + if initialAgent == agentB { + expectedAlternate = agentA + } + + // Never acknowledge creation: past AGENT_PENDING_REQUEST_TIMEOUT the + // sweep must exclude initialAgent, re-evaluate, and land on the only + // other ready agent - with a fresh pending cycle, not stuck/failed. + // Polled as a single (agent, status) pair rather than two separate + // Eventually calls, so a transient in-between state can't pass either + // check on its own. + type instanceState struct { + AgentName string + Status string + } + Eventually(func(g Gomega) instanceState { + getResp, err := rmApiClient.GetInstanceWithResponse(context.Background(), instanceID, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) + var st instanceState + if getResp.JSON200.AgentName != nil { + st.AgentName = *getResp.JSON200.AgentName + } + if getResp.JSON200.Status != nil { + st.Status = *getResp.JSON200.Status + } + return st + }).WithTimeout(30 * time.Second).WithPolling(1 * time.Second). + Should(Equal(instanceState{AgentName: expectedAlternate, Status: "pending"})) + }) +}) + +// Sibling reassignment: reassignExcludedSiblings (placement.go) proactively +// reassigns a run-sibling still pointed at an excluded agent instead of +// waiting for its own independent sweep timeout. The first three tests +// exercise that path directly against the real NATS-driven sweep and the +// real ReassignAndReset CAS; the last is a single-resource complement +// covering the sweep's retry-exhaustion path (see its own comment) - see +// PR #37 review thread r3761347505. +var _ = Describe("Sibling reassignment during self-heal", func() { + BeforeEach(func() { + resetWireMock() + }) + + It("proactively reassigns a cancelled run-sibling stuck on the excluded agent", func() { + agentA := registerReadyAgent("vm") + agentB := registerReadyAgent("vm") + policyID := createTwoAgentPolicy(agentA, agentB) + DeferCleanup(func() { + _, _ = policyApiClient.DeletePolicyWithResponse(context.Background(), policyID) + }) + + catalogItemID := createSiblingCatalogItem( + siblingResource{Name: "primary", ServiceType: "vm"}, + siblingResource{Name: "sibling", ServiceType: "vm"}, + ) + createCatalogItemInstance(catalogItemID) + + // createTwoAgentPolicy deterministically prefers agentA, so both + // same-service-type siblings land there synchronously as part of + // instance creation (CreateRun provisions dag_level-0 resources + // inline, before the instance-creation call returns). + initialIDs := listInstanceIDsByAgent(Default, agentA) + Expect(initialIDs).To(HaveLen(2)) + primaryID, siblingID := initialIDs[0], initialIDs[1] + + // Drive the sibling to "cancelled" via the real queued -> cancelled + // event path, without ever touching the primary. Leaving it + // "pending" instead would be ambiguous: both siblings start pending + // at essentially the same instant, so sweepPending's own per-row + // query would independently heal a still-pending sibling in the + // same tick it catches the primary, reaching the same end state + // even with reassignExcludedSiblings deleted entirely. "cancelled", + // unlike "pending", is never independently revisited by either + // sweep (sweepPending only scans status=pending; sweepQueued only + // scans status=queued), so the sibling can *only* move again via + // reassignExcludedSiblings's CAS, which explicitly allows pending + // OR cancelled. + // This is racing the primary's 5s pending timeout: if the sibling + // were still "queued" (not yet "cancelled") when the primary's sweep + // fires, reassignExcludedSiblings would skip it that tick, since + // "queued" isn't CAS-eligible either. In a healthy stack these two + // event round-trips settle in well under a second, so this isn't + // expected to flake; it would only surface if the consumer were + // already failing to process events (e.g. a DB error triggering the + // nak-and-redeliver path in handleRequestQueued), which would be a + // real bug worth surfacing as a failure anyway. + publishResponseEvent(messaging.CETypeRequestQueued, siblingID, agentA) + eventuallyStatus(siblingID).Should(Equal("queued")) + publishResponseEvent(messaging.CETypeCancelAcknowledged, siblingID, agentA) + eventuallyStatus(siblingID).Should(Equal("cancelled")) + + // Never acknowledge the primary: past the pending timeout the sweep + // excludes agentA for it, reassigns it to agentB, and + // reassignExcludedSiblings proactively reassigns the cancelled + // sibling too - both land on agentB with a fresh pending cycle, not + // stuck on agentA or split across agents. + Eventually(func(g Gomega) []string { + return listInstanceIDsByAgent(g, agentB) + }).WithTimeout(30 * time.Second).WithPolling(1 * time.Second).Should(ConsistOf(primaryID, siblingID)) + Expect(listInstanceIDsByAgent(Default, agentA)).To(BeEmpty()) + + for _, id := range []string{primaryID, siblingID} { + getResp, err := rmApiClient.GetInstanceWithResponse(context.Background(), id, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) + Expect(getResp.JSON200.Status).NotTo(BeNil()) + Expect(*getResp.JSON200.Status).To(Equal("pending")) + } + }) + + It("leaves an already-provisioning sibling on the excluded agent untouched", func() { + agentA := registerReadyAgent("vm") + agentB := registerReadyAgent("vm") + policyID := createTwoAgentPolicy(agentA, agentB) + DeferCleanup(func() { + _, _ = policyApiClient.DeletePolicyWithResponse(context.Background(), policyID) + }) + + catalogItemID := createSiblingCatalogItem( + siblingResource{Name: "primary", ServiceType: "vm"}, + siblingResource{Name: "sibling", ServiceType: "vm"}, + ) + createCatalogItemInstance(catalogItemID) + + initialIDs := listInstanceIDsByAgent(Default, agentA) + Expect(initialIDs).To(HaveLen(2)) + ackedID, stillPendingID := initialIDs[0], initialIDs[1] + + // Move one sibling to "provisioning" - no longer CAS-eligible for + // ReassignAndReset - before the pending timeout fires; leave the + // other pending. + publishResponseEvent(messaging.CETypeCreationAcknowledged, ackedID, agentA) + eventuallyStatus(ackedID).Should(Equal("provisioning")) + + // Past the pending timeout: sweepPending only ever picks up + // stillPendingID (ackedID no longer matches its status=pending + // filter), which self-heals to agentB. Whether or not + // reassignExcludedSiblings also attempts ackedID as part of that + // (its CAS would reject it, since it's no longer pending/cancelled), + // the observable guarantee this asserts - a provisioning instance is + // never reassigned by self-heal - holds either way. + Eventually(func(g Gomega) []string { + return listInstanceIDsByAgent(g, agentB) + }).WithTimeout(30 * time.Second).WithPolling(1 * time.Second).Should(ConsistOf(stillPendingID)) + + getResp, err := rmApiClient.GetInstanceWithResponse(context.Background(), ackedID, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) + Expect(getResp.JSON200.AgentName).NotTo(BeNil()) + Expect(*getResp.JSON200.AgentName).To(Equal(agentA)) + Expect(getResp.JSON200.Status).NotTo(BeNil()) + Expect(*getResp.JSON200.Status).To(Equal("provisioning")) + }) + + It("leaves a sibling on a different, never-excluded agent untouched", func() { + agentA := registerReadyAgent("vm") + agentB := registerReadyAgent("vm") + agentC := registerReadyAgent("database") + policyID := createThreeAgentPolicy(agentA, agentB, agentC) + DeferCleanup(func() { + _, _ = policyApiClient.DeletePolicyWithResponse(context.Background(), policyID) + }) + + catalogItemID := createSiblingCatalogItem( + siblingResource{Name: "primary", ServiceType: "vm"}, + siblingResource{Name: "sibling", ServiceType: "database"}, + ) + createCatalogItemInstance(catalogItemID) + + primaryID := findInstanceByAgentAndServiceType(Default, agentA, "vm") + siblingID := findInstanceByAgentAndServiceType(Default, agentC, "database") + + // Never acknowledge either. Past the pending timeout, the primary + // (on excluded agentA) moves to agentB. + Eventually(func(g Gomega) *string { + getResp, err := rmApiClient.GetInstanceWithResponse(context.Background(), primaryID, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) + return getResp.JSON200.AgentName + }).WithTimeout(30 * time.Second).WithPolling(1 * time.Second).Should(HaveValue(Equal(agentB))) + + // The sibling's agent_name never moves off agentC, during the + // primary's reassignment window or after: reassignExcludedSiblings + // is the only thing in this flow that could move it, and agentC + // was never in the excluded set. (Its own independent pending + // timeout may separately drive its *status* toward "failed" once + // retries exhaust, since agentC is the only database-capable + // agent - that's covered by the retries-exhausted test below and + // is irrelevant to the agent_name assertion here.) + Consistently(func(g Gomega) *string { + getResp, err := rmApiClient.GetInstanceWithResponse(context.Background(), siblingID, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) + return getResp.JSON200.AgentName + }).WithTimeout(5 * time.Second).WithPolling(1 * time.Second).Should(HaveValue(Equal(agentC))) + }) + + It("marks a pending instance failed once retries are exhausted with no alternate agent", func() { + agentA := registerReadyAgent("vm") + policyID := createAgentSelectingPolicy(agentA) + DeferCleanup(func() { + _, _ = policyApiClient.DeletePolicyWithResponse(context.Background(), policyID) + }) + + catalogItemID := createTestCatalogItem("vm") + createCatalogItemInstance(catalogItemID) + + instanceID := findInstanceByAgentName(agentA) + + // Never acknowledge: with no alternate agent, every self-heal + // attempt fails. This is a single, non-sibling resource - it + // exercises the sweep's own retry-exhaustion -> markFailedFrom path + // end-to-end (real DB CAS, real NATS-driven sweep), not the sibling + // mechanism the other tests in this Describe target; sweep_test.go + // covers the exhaustion timing/count logic at the unit level. + Eventually(func(g Gomega) string { + getResp, err := rmApiClient.GetInstanceWithResponse(context.Background(), instanceID, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) + g.Expect(getResp.JSON200.Status).NotTo(BeNil()) + return *getResp.JSON200.Status + }).WithTimeout(45 * time.Second).WithPolling(1 * time.Second).Should(Equal("failed")) + }) +}) diff --git a/test/subsystem/sp/service_instance_test.go b/test/subsystem/sp/service_instance_test.go index 38e2432..7251228 100644 --- a/test/subsystem/sp/service_instance_test.go +++ b/test/subsystem/sp/service_instance_test.go @@ -8,70 +8,18 @@ import ( "os" "time" - providerapi "github.com/dcm-project/control-plane/api/sp/v1alpha1/provider" "github.com/dcm-project/control-plane/api/sp/v1alpha1/resource_manager" - providerclient "github.com/dcm-project/control-plane/pkg/sp/client/provider" - rmClient "github.com/dcm-project/control-plane/pkg/sp/client/resource_manager" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) var _ = Describe("Service Instance API", func() { - var ( - rmApiClient *rmClient.ClientWithResponses - apiClient *providerclient.ClientWithResponses - ctx context.Context - providerID string - providerName string - ) + var ctx context.Context BeforeEach(func() { - baseURL := os.Getenv("API_URL") - if baseURL == "" { - baseURL = "http://localhost:8080/api/v1alpha1" - } - - var err error - authEditor := rmClient.WithRequestEditorFn(func(_ context.Context, req *http.Request) error { - req.Header.Set("X-Forwarded-User", "test-admin-sub") - return nil - }) - rmApiClient, err = rmClient.NewClientWithResponses(baseURL, authEditor) - Expect(err).NotTo(HaveOccurred()) - - providerAuthEditor := providerclient.WithRequestEditorFn(func(_ context.Context, req *http.Request) error { - req.Header.Set("X-Forwarded-User", "test-admin-sub") - return nil - }) - apiClient, err = providerclient.NewClientWithResponses(baseURL, providerAuthEditor) - Expect(err).NotTo(HaveOccurred()) - ctx = context.Background() - resetWireMock() - stubProviderHealthEndpoint() - stubProviderCreateInstance() - stubProviderDeleteInstance() - - providerName = "e2e-provider-" + uuid.New().String()[:8] - createResp, err := apiClient.CreateProviderWithResponse(ctx, nil, providerapi.Provider{ - Name: providerName, - Endpoint: providerEndpoint(), - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(createResp.StatusCode()).To(Equal(http.StatusCreated)) - providerID = *createResp.JSON201.Id - - waitForProviderReady(apiClient, ctx, providerID) - }) - - AfterEach(func() { - if providerID != "" { - apiClient.DeleteProviderWithResponse(ctx, providerID) - } }) Describe("Health Check", func() { @@ -89,104 +37,43 @@ var _ = Describe("Service Instance API", func() { }) Describe("Create Instance", func() { - It("creates an instance with specified ID", func() { - instID := uuid.New().String() - params := &resource_manager.CreateInstanceParams{Id: &instID} - createResp, err := rmApiClient.CreateInstanceWithResponse(ctx, params, resource_manager.ServiceTypeInstance{ - ProviderName: providerName, - Spec: map[string]interface{}{"cpu": 2, "memory": "4GB", "service_type": "vm"}, - }) - - Expect(err).NotTo(HaveOccurred()) - Expect(createResp.StatusCode()).To(Equal(http.StatusCreated)) - Expect(createResp.JSON201).NotTo(BeNil()) - Expect(*createResp.JSON201.Id).To(Equal(instID)) - Expect(createResp.JSON201.ProviderName).To(Equal(providerName)) - }) - - It("creates an instance with server-generated ID", func() { - createResp, err := rmApiClient.CreateInstanceWithResponse(ctx, nil, resource_manager.ServiceTypeInstance{ - ProviderName: providerName, - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, - }) + // The direct POST /service-type-instances endpoint never receives an + // agent_name from the caller - it's resolved upstream by policy - so + // it always 400s (see internal/sp/handlers/resource_manager and its + // unit tests). Successful creation is only reachable via the + // catalog -> placement -> policy -> SPRM agent-routed path, which is + // what these tests exercise. + It("creates an instance through the agent-routed path with agent_name populated", func() { + agentName, instanceID := createInstanceViaAgent() + getResp, err := rmApiClient.GetInstanceWithResponse(ctx, instanceID, nil) Expect(err).NotTo(HaveOccurred()) - Expect(createResp.StatusCode()).To(Equal(http.StatusCreated)) - Expect(createResp.JSON201.Id).NotTo(BeNil()) - Expect(*createResp.JSON201.Id).NotTo(BeEmpty()) - }) - - It("returns 409 for duplicate instance ID", func() { - instID := uuid.New().String() - params := &resource_manager.CreateInstanceParams{Id: &instID} - body := resource_manager.ServiceTypeInstance{ - ProviderName: providerName, - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, - } - - resp1, err := rmApiClient.CreateInstanceWithResponse(ctx, params, body) - Expect(err).NotTo(HaveOccurred()) - Expect(resp1.StatusCode()).To(Equal(http.StatusCreated)) - - resp2, err := rmApiClient.CreateInstanceWithResponse(ctx, params, body) - Expect(err).NotTo(HaveOccurred()) - Expect(resp2.StatusCode()).To(Equal(http.StatusConflict)) - }) - - It("returns 404 for non-existent provider", func() { - createResp, err := rmApiClient.CreateInstanceWithResponse(ctx, nil, resource_manager.ServiceTypeInstance{ - ProviderName: "non-existent-provider-" + uuid.New().String(), - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, - }) - - Expect(err).NotTo(HaveOccurred()) - Expect(createResp.StatusCode()).To(Equal(http.StatusNotFound)) + Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) + Expect(getResp.JSON200).NotTo(BeNil()) + Expect(getResp.JSON200.AgentName).NotTo(BeNil()) + Expect(*getResp.JSON200.AgentName).To(Equal(agentName)) }) - It("returns 422 when provider health status is not Ready", func() { - unhealthyName := "unhealthy-provider-" + uuid.New().String()[:8] - createProviderResp, err := apiClient.CreateProviderWithResponse(ctx, nil, providerapi.Provider{ - Name: unhealthyName, - Endpoint: "http://invalid-endpoint-does-not-exist.local/api", - ServiceType: "vm", - SchemaVersion: "v1alpha1", - }) - Expect(err).NotTo(HaveOccurred()) - Expect(createProviderResp.StatusCode()).To(Equal(http.StatusCreated)) - unhealthyProviderID := *createProviderResp.JSON201.Id - - defer func() { - apiClient.DeleteProviderWithResponse(ctx, unhealthyProviderID) - }() - + It("returns 400 because the direct endpoint never supplies an agent name", func() { createResp, err := rmApiClient.CreateInstanceWithResponse(ctx, nil, resource_manager.ServiceTypeInstance{ - ProviderName: unhealthyName, - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, + Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, }) Expect(err).NotTo(HaveOccurred()) - Expect(createResp.StatusCode()).To(Equal(http.StatusUnprocessableEntity)) + Expect(createResp.StatusCode()).To(Equal(http.StatusBadRequest)) }) }) Describe("Get Instance", func() { It("returns 200 for existing instance", func() { - instID := uuid.New().String() - params := &resource_manager.CreateInstanceParams{Id: &instID} - createResp, err := rmApiClient.CreateInstanceWithResponse(ctx, params, resource_manager.ServiceTypeInstance{ - ProviderName: providerName, - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(createResp.StatusCode()).To(Equal(http.StatusCreated)) + _, instanceID := createInstanceViaAgent() - getResp, err := rmApiClient.GetInstanceWithResponse(ctx, instID, nil) + getResp, err := rmApiClient.GetInstanceWithResponse(ctx, instanceID, nil) Expect(err).NotTo(HaveOccurred()) Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) Expect(getResp.JSON200).NotTo(BeNil()) - Expect(*getResp.JSON200.Id).To(Equal(instID)) - Expect(getResp.JSON200.ProviderName).To(Equal(providerName)) + Expect(*getResp.JSON200.Id).To(Equal(instanceID)) }) It("returns 404 for non-existent instance", func() { @@ -200,16 +87,9 @@ var _ = Describe("Service Instance API", func() { Describe("List Instances", func() { It("returns created instances in the list", func() { - instID := uuid.New().String() - params := &resource_manager.CreateInstanceParams{Id: &instID} - createResp, err := rmApiClient.CreateInstanceWithResponse(ctx, params, resource_manager.ServiceTypeInstance{ - ProviderName: providerName, - Spec: map[string]interface{}{"cpu": 1, "service_type": "vm"}, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(createResp.StatusCode()).To(Equal(http.StatusCreated)) + agentName, instanceID := createInstanceViaAgent() - listResp, err := rmApiClient.ListInstancesWithResponse(ctx, nil) + listResp, err := rmApiClient.ListInstancesWithResponse(ctx, &resource_manager.ListInstancesParams{AgentName: &agentName}) Expect(err).NotTo(HaveOccurred()) Expect(listResp.StatusCode()).To(Equal(http.StatusOK)) @@ -219,20 +99,7 @@ var _ = Describe("Service Instance API", func() { for i, inst := range *listResp.JSON200.Instances { ids[i] = *inst.Id } - Expect(ids).To(ContainElement(instID)) - }) - - It("filters by provider name", func() { - filterProvider := providerName - params := &resource_manager.ListInstancesParams{ - Provider: &filterProvider, - } - - listResp, err := rmApiClient.ListInstancesWithResponse(ctx, params) - - Expect(err).NotTo(HaveOccurred()) - Expect(listResp.StatusCode()).To(Equal(http.StatusOK)) - Expect(listResp.JSON200).NotTo(BeNil()) + Expect(ids).To(ContainElement(instanceID)) }) It("respects max page size parameter", func() { @@ -277,21 +144,14 @@ var _ = Describe("Service Instance API", func() { Describe("Delete Instance", func() { It("returns 204 and instance is removed", func() { - instID := uuid.New().String() - params := &resource_manager.CreateInstanceParams{Id: &instID} - createResp, err := rmApiClient.CreateInstanceWithResponse(ctx, params, resource_manager.ServiceTypeInstance{ - ProviderName: providerName, - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(createResp.StatusCode()).To(Equal(http.StatusCreated)) + _, instanceID := createInstanceViaAgent() - deleteResp, err := rmApiClient.DeleteInstanceWithResponse(ctx, instID, nil) + deleteResp, err := rmApiClient.DeleteInstanceWithResponse(ctx, instanceID, nil) Expect(err).NotTo(HaveOccurred()) Expect(deleteResp.StatusCode()).To(Equal(http.StatusNoContent)) - getResp, err := rmApiClient.GetInstanceWithResponse(ctx, instID, nil) + getResp, err := rmApiClient.GetInstanceWithResponse(ctx, instanceID, nil) Expect(err).NotTo(HaveOccurred()) Expect(getResp.StatusCode()).To(Equal(http.StatusNotFound)) }) @@ -306,64 +166,13 @@ var _ = Describe("Service Instance API", func() { }) Describe("Deferred Deletion", func() { - var instID string + var instID, agentName string createInstance := func() { - instID = uuid.New().String() - params := &resource_manager.CreateInstanceParams{Id: &instID} - createResp, err := rmApiClient.CreateInstanceWithResponse(ctx, params, resource_manager.ServiceTypeInstance{ - ProviderName: providerName, - Spec: map[string]interface{}{"cpu": 2, "service_type": "vm"}, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(createResp.StatusCode()).To(Equal(http.StatusCreated)) + agentName, instID = createInstanceViaAgent() } - It("returns error on non-deferred delete when SP fails", func() { - createInstance() - clearDeleteStubAndStubFailure() - - deleteResp, err := rmApiClient.DeleteInstanceWithResponse(ctx, instID, nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(deleteResp.StatusCode()).To(Equal(http.StatusInternalServerError)) - - // Instance should still be active - getResp, err := rmApiClient.GetInstanceWithResponse(ctx, instID, nil) - Expect(err).NotTo(HaveOccurred()) - Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) - Expect(getResp.JSON200.DeletionStatus).To(BeNil()) - }) - - It("returns 204 on deferred delete when SP fails and marks instance SCHEDULED", func() { - createInstance() - clearDeleteStubAndStubFailure() - - deferred := true - deleteResp, err := rmApiClient.DeleteInstanceWithResponse(ctx, instID, &resource_manager.DeleteInstanceParams{ - Deferred: &deferred, - }) - - Expect(err).NotTo(HaveOccurred()) - Expect(deleteResp.StatusCode()).To(Equal(http.StatusNoContent)) - - // Instance should be hidden from default GET - getResp, err := rmApiClient.GetInstanceWithResponse(ctx, instID, nil) - Expect(err).NotTo(HaveOccurred()) - Expect(getResp.StatusCode()).To(Equal(http.StatusNotFound)) - - // Instance should be visible with show_deleted=true - showDeleted := true - getResp, err = rmApiClient.GetInstanceWithResponse(ctx, instID, &resource_manager.GetInstanceParams{ - ShowDeleted: &showDeleted, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) - Expect(getResp.JSON200.DeletionStatus).NotTo(BeNil()) - Expect(*getResp.JSON200.DeletionStatus).To(Equal(resource_manager.SCHEDULED)) - }) - - It("marks instance SCHEDULED on deferred delete even when SP is available", func() { + It("marks instance SCHEDULED on deferred delete", func() { createInstance() deferred := true @@ -374,19 +183,18 @@ var _ = Describe("Service Instance API", func() { Expect(err).NotTo(HaveOccurred()) Expect(deleteResp.StatusCode()).To(Equal(http.StatusNoContent)) - // Instance should be marked SCHEDULED, not hard-deleted showDeleted := true getResp, err := rmApiClient.GetInstanceWithResponse(ctx, instID, &resource_manager.GetInstanceParams{ ShowDeleted: &showDeleted, }) Expect(err).NotTo(HaveOccurred()) Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) + Expect(getResp.JSON200.DeletionStatus).NotTo(BeNil()) Expect(string(*getResp.JSON200.DeletionStatus)).To(Equal("SCHEDULED")) }) It("excludes soft-deleted instances from default LIST", func() { createInstance() - clearDeleteStubAndStubFailure() deferred := true deleteResp, err := rmApiClient.DeleteInstanceWithResponse(ctx, instID, &resource_manager.DeleteInstanceParams{ @@ -395,8 +203,7 @@ var _ = Describe("Service Instance API", func() { Expect(err).NotTo(HaveOccurred()) Expect(deleteResp.StatusCode()).To(Equal(http.StatusNoContent)) - // Default LIST should not include the soft-deleted instance - listResp, err := rmApiClient.ListInstancesWithResponse(ctx, nil) + listResp, err := rmApiClient.ListInstancesWithResponse(ctx, &resource_manager.ListInstancesParams{AgentName: &agentName}) Expect(err).NotTo(HaveOccurred()) Expect(listResp.StatusCode()).To(Equal(http.StatusOK)) @@ -409,7 +216,6 @@ var _ = Describe("Service Instance API", func() { It("includes soft-deleted instances in LIST with show_deleted=true", func() { createInstance() - clearDeleteStubAndStubFailure() deferred := true deleteResp, err := rmApiClient.DeleteInstanceWithResponse(ctx, instID, &resource_manager.DeleteInstanceParams{ @@ -418,10 +224,10 @@ var _ = Describe("Service Instance API", func() { Expect(err).NotTo(HaveOccurred()) Expect(deleteResp.StatusCode()).To(Equal(http.StatusNoContent)) - // LIST with show_deleted=true should include the soft-deleted instance showDeleted := true listResp, err := rmApiClient.ListInstancesWithResponse(ctx, &resource_manager.ListInstancesParams{ ShowDeleted: &showDeleted, + AgentName: &agentName, }) Expect(err).NotTo(HaveOccurred()) Expect(listResp.StatusCode()).To(Equal(http.StatusOK)) @@ -434,11 +240,9 @@ var _ = Describe("Service Instance API", func() { Expect(ids).To(ContainElement(instID)) }) - It("transitions to PENDING_PROVIDER when provider goes down and back to SCHEDULED on recovery", func() { + It("can hard-delete a soft-deleted instance", func() { createInstance() - clearDeleteStubAndStubFailure() - // Deferred delete → SCHEDULED deferred := true deleteResp, err := rmApiClient.DeleteInstanceWithResponse(ctx, instID, &resource_manager.DeleteInstanceParams{ Deferred: &deferred, @@ -446,153 +250,25 @@ var _ = Describe("Service Instance API", func() { Expect(err).NotTo(HaveOccurred()) Expect(deleteResp.StatusCode()).To(Equal(http.StatusNoContent)) - showDeleted := true - getResp, err := rmApiClient.GetInstanceWithResponse(ctx, instID, &resource_manager.GetInstanceParams{ - ShowDeleted: &showDeleted, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) - Expect(*getResp.JSON200.DeletionStatus).To(Equal(resource_manager.SCHEDULED)) - - // Remove health stub → provider becomes NotReady - resetHealthStubs() - waitForProviderNotReady(apiClient, ctx, providerID) - - // Instance should transition to PENDING_PROVIDER - Eventually(func() resource_manager.ServiceTypeInstanceDeletionStatus { - resp, err := rmApiClient.GetInstanceWithResponse(ctx, instID, &resource_manager.GetInstanceParams{ - ShowDeleted: &showDeleted, - }) - if err != nil || resp.StatusCode() != http.StatusOK || resp.JSON200 == nil || resp.JSON200.DeletionStatus == nil { - return "" - } - return *resp.JSON200.DeletionStatus - }, 30*time.Second, 1*time.Second).Should(Equal(resource_manager.PENDINGPROVIDER)) - - // Restore health stub → provider recovers - stubProviderHealthEndpoint() - waitForProviderReady(apiClient, ctx, providerID) - - // Instance should transition back to SCHEDULED - Eventually(func() resource_manager.ServiceTypeInstanceDeletionStatus { - resp, err := rmApiClient.GetInstanceWithResponse(ctx, instID, &resource_manager.GetInstanceParams{ - ShowDeleted: &showDeleted, - }) - if err != nil || resp.StatusCode() != http.StatusOK || resp.JSON200 == nil || resp.JSON200.DeletionStatus == nil { - return "" - } - return *resp.JSON200.DeletionStatus - }, 30*time.Second, 1*time.Second).Should(Equal(resource_manager.SCHEDULED)) - - // Clean up: restore delete stub and hard-delete - resetDeleteStubs() - stubProviderDeleteInstance() - deleteResp, err = rmApiClient.DeleteInstanceWithResponse(ctx, instID, nil) Expect(err).NotTo(HaveOccurred()) Expect(deleteResp.StatusCode()).To(Equal(http.StatusNoContent)) - getResp, err = rmApiClient.GetInstanceWithResponse(ctx, instID, &resource_manager.GetInstanceParams{ - ShowDeleted: &showDeleted, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(getResp.StatusCode()).To(Equal(http.StatusNotFound)) - }) - - It("transitions to PENDING_PROVIDER when provider reports unhealthy and back to SCHEDULED on recovery", func() { - createInstance() - clearDeleteStubAndStubFailure() - - // Deferred delete → SCHEDULED - deferred := true - deleteResp, err := rmApiClient.DeleteInstanceWithResponse(ctx, instID, &resource_manager.DeleteInstanceParams{ - Deferred: &deferred, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(deleteResp.StatusCode()).To(Equal(http.StatusNoContent)) + // A non-deferred delete of an agent-routed instance only + // completes once the agent's deletion-acknowledged event + // arrives (see service_type_instance.go's DeleteInstance / + // consumer.ResponseConsumer.handleDeletionAcknowledged); with + // no live agent in this stack, simulate that confirmation. + acknowledgeDeletion(instID, agentName) showDeleted := true - getResp, err := rmApiClient.GetInstanceWithResponse(ctx, instID, &resource_manager.GetInstanceParams{ - ShowDeleted: &showDeleted, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(getResp.StatusCode()).To(Equal(http.StatusOK)) - Expect(*getResp.JSON200.DeletionStatus).To(Equal(resource_manager.SCHEDULED)) - - // Stub unhealthy response → provider becomes Unhealthy - stubProviderHealthUnhealthy() - waitForProviderUnhealthy(apiClient, ctx, providerID) - - // Instance should transition to PENDING_PROVIDER - Eventually(func() resource_manager.ServiceTypeInstanceDeletionStatus { - resp, err := rmApiClient.GetInstanceWithResponse(ctx, instID, &resource_manager.GetInstanceParams{ + Eventually(func(g Gomega) int { + getResp, err := rmApiClient.GetInstanceWithResponse(ctx, instID, &resource_manager.GetInstanceParams{ ShowDeleted: &showDeleted, }) - if err != nil || resp.StatusCode() != http.StatusOK || resp.JSON200 == nil || resp.JSON200.DeletionStatus == nil { - return "" - } - return *resp.JSON200.DeletionStatus - }, 30*time.Second, 1*time.Second).Should(Equal(resource_manager.PENDINGPROVIDER)) - - // Restore healthy stub → provider recovers - resetHealthStubs() - stubProviderHealthEndpoint() - waitForProviderReady(apiClient, ctx, providerID) - - // Instance should transition back to SCHEDULED - Eventually(func() resource_manager.ServiceTypeInstanceDeletionStatus { - resp, err := rmApiClient.GetInstanceWithResponse(ctx, instID, &resource_manager.GetInstanceParams{ - ShowDeleted: &showDeleted, - }) - if err != nil || resp.StatusCode() != http.StatusOK || resp.JSON200 == nil || resp.JSON200.DeletionStatus == nil { - return "" - } - return *resp.JSON200.DeletionStatus - }, 30*time.Second, 1*time.Second).Should(Equal(resource_manager.SCHEDULED)) - - // Clean up: restore delete stub and hard-delete - resetDeleteStubs() - stubProviderDeleteInstance() - - deleteResp, err = rmApiClient.DeleteInstanceWithResponse(ctx, instID, nil) - Expect(err).NotTo(HaveOccurred()) - Expect(deleteResp.StatusCode()).To(Equal(http.StatusNoContent)) - - getResp, err = rmApiClient.GetInstanceWithResponse(ctx, instID, &resource_manager.GetInstanceParams{ - ShowDeleted: &showDeleted, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(getResp.StatusCode()).To(Equal(http.StatusNotFound)) - }) - - It("can re-delete a soft-deleted instance when SP becomes available", func() { - createInstance() - clearDeleteStubAndStubFailure() - - // First delete: deferred, SP fails -> mark SCHEDULED - deferred := true - deleteResp, err := rmApiClient.DeleteInstanceWithResponse(ctx, instID, &resource_manager.DeleteInstanceParams{ - Deferred: &deferred, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(deleteResp.StatusCode()).To(Equal(http.StatusNoContent)) - - // Restore SP delete stub to succeed - resetDeleteStubs() - stubProviderDeleteInstance() - - // Second delete: should hard-delete the SCHEDULED instance - deleteResp, err = rmApiClient.DeleteInstanceWithResponse(ctx, instID, nil) - Expect(err).NotTo(HaveOccurred()) - Expect(deleteResp.StatusCode()).To(Equal(http.StatusNoContent)) - - // Instance should be fully gone - showDeleted := true - getResp, err := rmApiClient.GetInstanceWithResponse(ctx, instID, &resource_manager.GetInstanceParams{ - ShowDeleted: &showDeleted, - }) - Expect(err).NotTo(HaveOccurred()) - Expect(getResp.StatusCode()).To(Equal(http.StatusNotFound)) + g.Expect(err).NotTo(HaveOccurred()) + return getResp.StatusCode() + }).WithTimeout(10 * time.Second).WithPolling(500 * time.Millisecond).Should(Equal(http.StatusNotFound)) }) }) }) diff --git a/test/subsystem/sp/setup_test.go b/test/subsystem/sp/setup_test.go index 9d3eaf8..9feb105 100644 --- a/test/subsystem/sp/setup_test.go +++ b/test/subsystem/sp/setup_test.go @@ -3,17 +3,40 @@ package subsystem_test import ( - "bytes" "context" + _ "embed" "encoding/json" "net/http" "os" - "time" + "strings" - providerclient "github.com/dcm-project/control-plane/pkg/sp/client/provider" + agentapi "github.com/dcm-project/control-plane/api/agent/v1alpha1" + catalogapi "github.com/dcm-project/control-plane/api/catalog/v1alpha1" + policyapi "github.com/dcm-project/control-plane/api/policy/v1alpha1" + "github.com/dcm-project/control-plane/api/sp/v1alpha1/resource_manager" + "github.com/dcm-project/control-plane/internal/catalog/testutil" + "github.com/dcm-project/control-plane/internal/sp/messaging" + "github.com/google/uuid" + "github.com/nats-io/nats.go" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) +// agentSelectingPolicyRego, twoAgentSelectingPolicyRego, and +// threeAgentSelectingPolicyRego are Rego policy templates for the test-data +// below, with placeholder agent names substituted in at runtime. See +// createAgentSelectingPolicy, createTwoAgentPolicy, and createThreeAgentPolicy. +var ( + //go:embed testdata/agent_selecting_policy.rego + agentSelectingPolicyRego string + + //go:embed testdata/two_agent_selecting_policy.rego + twoAgentSelectingPolicyRego string + + //go:embed testdata/three_agent_selecting_policy.rego + threeAgentSelectingPolicyRego string +) + func wireMockURL() string { if url := os.Getenv("WIREMOCK_URL"); url != "" { return url @@ -21,197 +44,342 @@ func wireMockURL() string { return "http://localhost:9090" } -// providerEndpoint returns the WireMock URL as seen from inside the container network. -func providerEndpoint() string { - if url := os.Getenv("PROVIDER_ENDPOINT"); url != "" { - return url - } - return "http://provider-wiremock:8080" -} - func resetWireMock() { req, _ := http.NewRequest(http.MethodDelete, wireMockURL()+"/__admin/mappings", nil) http.DefaultClient.Do(req) } -func stubProviderHealthEndpoint() { - stub := map[string]interface{}{ - "request": map[string]interface{}{ - "method": "GET", - "urlPath": "/health", - }, - "response": map[string]interface{}{ - "status": 200, - "headers": map[string]string{ - "Content-Type": "application/json", - }, - "jsonBody": map[string]interface{}{ - "status": "healthy", - }, +func natsURL() string { + if url := os.Getenv("NATS_URL"); url != "" { + return url + } + return "nats://localhost:4222" +} + +// publishResponseEvent simulates a real agent's response for resourceID, +// since this stack has no live agent to send one. It publishes directly to +// messaging.ResponseSubject - the same wire contract consumer.ResponseConsumer +// listens on - rather than going through internal/sp/messaging.Publisher, +// which is a control-plane outbound (create/delete/cancel request) client, +// not a stand-in for an agent's own response producer. +func publishResponseEvent(ceType, resourceID, agentName string) { + nc, err := nats.Connect(natsURL()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + defer nc.Close() + + envelope := map[string]any{ + "specversion": messaging.CESpecVersion, + "type": ceType, + "source": "sp-subsystem-test-agent", + "id": uuid.New().String(), + "data": map[string]string{ + "resource_id": resourceID, + "agent_name": agentName, }, } + data, err := json.Marshal(envelope) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) - body, _ := json.Marshal(stub) - http.Post(wireMockURL()+"/__admin/mappings", "application/json", bytes.NewReader(body)) + ExpectWithOffset(1, nc.Publish(messaging.ResponseSubject, data)).To(Succeed()) + ExpectWithOffset(1, nc.Flush()).To(Succeed()) } -// waitForProviderReady polls the provider API until the provider's health status is "ready". -// This ensures the health check monitor has confirmed the provider before tests proceed. -func waitForProviderReady(apiClient *providerclient.ClientWithResponses, ctx context.Context, providerID string) { - Eventually(func() string { - getResp, err := apiClient.GetProviderWithResponse(ctx, providerID) - if err != nil || getResp.StatusCode() != http.StatusOK || getResp.JSON200 == nil { - return "" - } - if getResp.JSON200.HealthStatus == nil { - return "" - } - return *getResp.JSON200.HealthStatus - }, 30*time.Second, 1*time.Second).Should(Equal("ready"), "Provider should become ready") +// acknowledgeDeletion simulates the real agent's confirmation that a +// resource's physical deletion is complete. +func acknowledgeDeletion(resourceID, agentName string) { + publishResponseEvent(messaging.CETypeDeletionAcknowledged, resourceID, agentName) } -func stubProviderHealthUnhealthy() { - resetHealthStubs() - stub := map[string]interface{}{ - "request": map[string]interface{}{ - "method": "GET", - "urlPath": "/health", - }, - "response": map[string]interface{}{ - "status": 200, - "headers": map[string]string{ - "Content-Type": "application/json", - }, - "jsonBody": map[string]interface{}{ - "status": "unhealthy", - }, - }, +func ptr[T any](v T) *T { + return &v +} + +// registerReadyAgent registers a new agent (unique name) supporting +// serviceType and returns its name. Registration marks the agent "ready" +// immediately, no heartbeat required. +func registerReadyAgent(serviceType string) string { + name := "sp-subsystem-agent-" + uuid.New().String()[:8] + body := agentapi.AgentRegistrationRequest{ + Name: name, + Environment: "test", + ServiceTypes: []string{serviceType}, + Cost: agentapi.AgentRegistrationRequestCostMedium, + TopicName: "dcm.agent." + name, } - body, _ := json.Marshal(stub) - http.Post(wireMockURL()+"/__admin/mappings", "application/json", bytes.NewReader(body)) + resp, err := agentApiClient.CreateAgentWithResponse(context.Background(), body) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, resp.StatusCode()).To(Equal(http.StatusCreated)) + + return name } -func stubProviderCreateInstance() { - stub := map[string]interface{}{ - "request": map[string]interface{}{ - "method": "POST", - "urlPath": "/", - }, - "response": map[string]interface{}{ - "status": 200, - "headers": map[string]string{ - "Content-Type": "application/json", - }, - "transformers": []string{"response-template"}, - "jsonBody": map[string]interface{}{ - "id": "{{request.query.id}}", - "status": "PROVISIONING", - }, - }, +// createAgentSelectingPolicy creates an enabled policy that deterministically +// selects agentName whenever it appears in available_agents. Returns the +// policy ID for later cleanup. +func createAgentSelectingPolicy(agentName string) string { + regoCode := strings.ReplaceAll(agentSelectingPolicyRego, "__AGENT_NAME__", agentName) + + body := policyapi.Policy{ + DisplayName: ptr("sp-subsystem agent selector " + agentName), + PolicyType: ptr(policyapi.GLOBAL), + Priority: ptr(int32(500)), + Enabled: ptr(true), + RegoCode: ptr(regoCode), } - body, _ := json.Marshal(stub) - http.Post(wireMockURL()+"/__admin/mappings", "application/json", bytes.NewReader(body)) + resp, err := policyApiClient.CreatePolicyWithResponse(context.Background(), &policyapi.CreatePolicyParams{}, body) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, resp.StatusCode()).To(Equal(http.StatusCreated)) + + return *resp.JSON201.Id } -func stubProviderDeleteInstance() { - stub := map[string]interface{}{ - "request": map[string]interface{}{ - "method": "DELETE", - "urlPathPattern": "/.*", - }, - "response": map[string]interface{}{ - "status": 204, - }, +// createTwoAgentPolicy creates an enabled policy that prefers agentA over +// agentB, picking whichever is still present in available_agents. Policy +// evaluation pre-filters exclude_agents out of available_agents before Rego +// runs, so once the sweep excludes agentA the same policy resolves to +// agentB - driving both initial routing and self-heal re-evaluation with one +// policy. The names are baked in explicitly (rather than picking +// available_agents[0]) because the shared test DB accumulates "ready" agents +// left behind by other specs across the suite run, and index 0 isn't +// guaranteed to be this test's own agent. Returns the policy ID for cleanup. +func createTwoAgentPolicy(agentA, agentB string) string { + regoCode := strings.NewReplacer( + "__AGENT_A__", agentA, + "__AGENT_B__", agentB, + ).Replace(twoAgentSelectingPolicyRego) + + body := policyapi.Policy{ + DisplayName: ptr("sp-subsystem two-agent selector"), + PolicyType: ptr(policyapi.GLOBAL), + Priority: ptr(int32(500)), + Enabled: ptr(true), + RegoCode: ptr(regoCode), } - body, _ := json.Marshal(stub) - http.Post(wireMockURL()+"/__admin/mappings", "application/json", bytes.NewReader(body)) + resp, err := policyApiClient.CreatePolicyWithResponse(context.Background(), &policyapi.CreatePolicyParams{}, body) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, resp.StatusCode()).To(Equal(http.StatusCreated)) + + return *resp.JSON201.Id } -func stubProviderDeleteInstanceFailure() { - stub := map[string]interface{}{ - "request": map[string]interface{}{ - "method": "DELETE", - "urlPathPattern": "/.*", - }, - "response": map[string]interface{}{ - "status": 500, - "headers": map[string]string{ - "Content-Type": "application/json", - }, - "jsonBody": map[string]interface{}{ - "error": "internal server error", - }, +// createThreeAgentPolicy extends createTwoAgentPolicy's prefer-A/fallback-B +// rules with an unconditional "prefer C" rule. Policy evaluation +// pre-filters available_agents by service-type capability before Rego +// runs, so as long as only agentC is registered for the service type under +// evaluation (e.g. "database"), the C rule only ever fires for that +// resource - no service-type branching needed inside Rego itself. Returns +// the policy ID for cleanup. +func createThreeAgentPolicy(agentA, agentB, agentC string) string { + regoCode := strings.NewReplacer( + "__AGENT_A__", agentA, + "__AGENT_B__", agentB, + "__AGENT_C__", agentC, + ).Replace(threeAgentSelectingPolicyRego) + + body := policyapi.Policy{ + DisplayName: ptr("sp-subsystem three-agent selector"), + PolicyType: ptr(policyapi.GLOBAL), + Priority: ptr(int32(500)), + Enabled: ptr(true), + RegoCode: ptr(regoCode), + } + + resp, err := policyApiClient.CreatePolicyWithResponse(context.Background(), &policyapi.CreatePolicyParams{}, body) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, resp.StatusCode()).To(Equal(http.StatusCreated)) + + return *resp.JSON201.Id +} + +// createTestCatalogItem creates a single-resource catalog item for +// serviceType with a single editable "vcpu.count" field, matching the +// seeded "vm" service type schema. +func createTestCatalogItem(serviceType string) string { + id := "sp-subsystem-ci-" + uuid.New().String()[:8] + editable := true + fields := []catalogapi.FieldConfiguration{{ + Path: "vcpu.count", + DisplayName: ptr("vCPU Count"), + Editable: &editable, + Default: float64(2), + ValidationSchema: &map[string]any{ + "type": "number", + "minimum": float64(1), + "maximum": float64(16), }, + }} + + params := &catalogapi.CreateCatalogItemParams{Id: &id} + body := catalogapi.CatalogItem{ + ApiVersion: ptr("v1alpha1"), + DisplayName: ptr("SP subsystem test item " + id), + Spec: testutil.PtrCatalogSpec(serviceType, fields), } - body, _ := json.Marshal(stub) - http.Post(wireMockURL()+"/__admin/mappings", "application/json", bytes.NewReader(body)) + resp, err := catalogApiClient.CreateCatalogItemWithResponse(context.Background(), params, body) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, resp.StatusCode()).To(Equal(http.StatusCreated)) + + return id } -func clearDeleteStubAndStubFailure() { - // Remove all existing mappings for DELETE and re-stub with failure - resetDeleteStubs() - stubProviderDeleteInstanceFailure() +type siblingResource struct { + Name string + ServiceType string } -func resetStubsByMethod(method string) { - resp, err := http.Get(wireMockURL() + "/__admin/mappings") - if err != nil { - return +// createSiblingCatalogItem builds a multi-resource CatalogItem, one entry +// per siblingResource, with no RequiresResources (so every resource lands +// at dag_level 0 and is provisioned concurrently, per assignDagLevels) and +// no Fields (CatalogResource.Fields is optional - buildResourceSpecFromFields's +// per-field defaulting loop is a no-op on an empty slice). Returns the +// catalog item ID. +func createSiblingCatalogItem(resources ...siblingResource) string { + // display_name is capped at 63 chars by the schema: keep both the ID + // and the prefix short enough that id's own length doesn't push the + // concatenated display_name over that limit. + id := "sp-subsystem-ci-sib-" + uuid.New().String()[:8] + apiResources := make([]catalogapi.CatalogResource, len(resources)) + for i, r := range resources { + apiResources[i] = catalogapi.CatalogResource{ + Name: r.Name, + ServiceType: r.ServiceType, + } + } + + params := &catalogapi.CreateCatalogItemParams{Id: &id} + body := catalogapi.CatalogItem{ + ApiVersion: ptr("v1alpha1"), + DisplayName: ptr("SP subsystem sibling item " + id), + Spec: &catalogapi.CatalogItemSpec{Resources: apiResources}, } - defer resp.Body.Close() - - var result struct { - Mappings []struct { - ID string `json:"id"` - Request struct { - Method string `json:"method"` - } `json:"request"` - } `json:"mappings"` + + resp, err := catalogApiClient.CreateCatalogItemWithResponse(context.Background(), params, body) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, resp.StatusCode()).To(Equal(http.StatusCreated)) + + return id +} + +// listInstanceIDsByAgent returns the IDs of every instance currently on +// agentName, for callers that only care about the ID *set* - e.g. a pair of +// same-service-type siblings sharing one agent, where individual identity +// doesn't matter as long as both move together. Takes a Gomega (Default +// outside a poll, the closure's g inside one) rather than asserting via the +// package-level ExpectWithOffset, so a transient API error inside an +// Eventually/Consistently triggers a normal retry instead of hard-failing +// the spec. +func listInstanceIDsByAgent(g Gomega, agentName string) []string { + listResp, err := rmApiClient.ListInstancesWithResponse(context.Background(), &resource_manager.ListInstancesParams{AgentName: &agentName}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(listResp.StatusCode()).To(Equal(http.StatusOK)) + if listResp.JSON200.Instances == nil { + return nil } - json.NewDecoder(resp.Body).Decode(&result) - for _, m := range result.Mappings { - if m.Request.Method == method { - req, _ := http.NewRequest(http.MethodDelete, wireMockURL()+"/__admin/mappings/"+m.ID, nil) - http.DefaultClient.Do(req) + ids := make([]string, 0, len(*listResp.JSON200.Instances)) + for _, inst := range *listResp.JSON200.Instances { + if inst.Id != nil { + ids = append(ids, *inst.Id) } } + return ids } -func resetDeleteStubs() { - resetStubsByMethod("DELETE") +// findInstanceByAgentAndServiceType looks up the single instance on +// agentName with serviceType, for siblings distinguished by service type +// rather than by ID set. See listInstanceIDsByAgent for why it takes a +// Gomega instead of using the package-level Expect. +func findInstanceByAgentAndServiceType(g Gomega, agentName, serviceType string) string { + listResp, err := rmApiClient.ListInstancesWithResponse(context.Background(), &resource_manager.ListInstancesParams{ + AgentName: &agentName, + ServiceType: &serviceType, + }) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(listResp.StatusCode()).To(Equal(http.StatusOK)) + g.Expect(listResp.JSON200.Instances).NotTo(BeNil()) + g.Expect(*listResp.JSON200.Instances).To(HaveLen(1), "expected exactly one %q instance on agent %q", serviceType, agentName) + + return *(*listResp.JSON200.Instances)[0].Id } -func resetHealthStubs() { - resetStubsByMethod("GET") +// createInstanceViaAgent drives instance creation through the real +// production path: register a ready agent, install a policy that routes to +// it, create a catalog item, and create a catalog item instance. Placement +// resolves the agent via policy and calls the in-process SPRM client with a +// real agent name - the only path CreateInstance's agent_name validation +// allows through. Returns the agent name and the resulting resource_manager +// instance ID (found by matching agent_name, since catalog item instances +// don't expose the underlying resource ID). +func createInstanceViaAgent() (agentName, instanceID string) { + const serviceType = "vm" + + agentName = registerReadyAgent(serviceType) + policyID := createAgentSelectingPolicy(agentName) + DeferCleanup(func() { + _, _ = policyApiClient.DeletePolicyWithResponse(context.Background(), policyID) + }) + + catalogItemID := createTestCatalogItem(serviceType) + + instID := "sp-subsystem-inst-" + uuid.New().String()[:8] + instParams := &catalogapi.CreateCatalogItemInstanceParams{Id: &instID} + instBody := catalogapi.CatalogItemInstance{ + ApiVersion: "v1alpha1", + DisplayName: instID, // display_name is capped at 63 chars by the schema + Spec: catalogapi.CatalogItemInstanceSpec{ + CatalogItemId: catalogItemID, + UserValues: []catalogapi.UserValue{}, + }, + } + + instResp, err := catalogApiClient.CreateCatalogItemInstanceWithResponse(context.Background(), instParams, instBody) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, instResp.StatusCode()).To(Equal(http.StatusCreated)) + + DeferCleanup(func() { + _, _ = catalogApiClient.DeleteCatalogItemInstanceWithResponse(context.Background(), instID) + }) + + return agentName, findInstanceByAgentName(agentName) } -func waitForProviderNotReady(apiClient *providerclient.ClientWithResponses, ctx context.Context, providerID string) { - Eventually(func() string { - getResp, err := apiClient.GetProviderWithResponse(ctx, providerID) - if err != nil || getResp.StatusCode() != http.StatusOK || getResp.JSON200 == nil { - return "" - } - if getResp.JSON200.HealthStatus == nil { - return "" - } - return *getResp.JSON200.HealthStatus - }, 60*time.Second, 1*time.Second).Should(Equal("unavailable"), "Provider should become unavailable") +// findInstanceByAgentName looks up the resource_manager instance for a +// freshly minted, per-test agent name via the API's own agent_name filter, +// rather than scanning a fixed-size page of ListInstances: the shared +// suite-wide DB accumulates instances across specs (ordered oldest-first by +// create_time), so a fresh instance can fall off a fixed first page and be +// misreported as "not found" instead of a real routing failure. +func findInstanceByAgentName(agentName string) string { + listResp, err := rmApiClient.ListInstancesWithResponse(context.Background(), &resource_manager.ListInstancesParams{AgentName: &agentName}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, listResp.StatusCode()).To(Equal(http.StatusOK)) + ExpectWithOffset(1, listResp.JSON200.Instances).NotTo(BeNil()) + ExpectWithOffset(1, *listResp.JSON200.Instances).To(HaveLen(1), "expected exactly one instance for agent %q", agentName) + + return *(*listResp.JSON200.Instances)[0].Id } -func waitForProviderUnhealthy(apiClient *providerclient.ClientWithResponses, ctx context.Context, providerID string) { - Eventually(func() string { - getResp, err := apiClient.GetProviderWithResponse(ctx, providerID) - if err != nil || getResp.StatusCode() != http.StatusOK || getResp.JSON200 == nil { - return "" - } - if getResp.JSON200.HealthStatus == nil { - return "" - } - return *getResp.JSON200.HealthStatus - }, 60*time.Second, 1*time.Second).Should(Equal("unhealthy"), "Provider should become unhealthy") +// findInstanceByEitherAgent is findInstanceByAgentName for the self-heal +// test, which doesn't know upfront which of its two freshly registered +// agents the policy actually selected. +func findInstanceByEitherAgent(agentA, agentB string) (instanceID, matchedAgent string) { + listResp, err := rmApiClient.ListInstancesWithResponse(context.Background(), &resource_manager.ListInstancesParams{AgentName: &agentA}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, listResp.StatusCode()).To(Equal(http.StatusOK)) + ExpectWithOffset(1, listResp.JSON200.Instances).NotTo(BeNil()) + if len(*listResp.JSON200.Instances) == 1 { + return *(*listResp.JSON200.Instances)[0].Id, agentA + } + + listResp, err = rmApiClient.ListInstancesWithResponse(context.Background(), &resource_manager.ListInstancesParams{AgentName: &agentB}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + ExpectWithOffset(1, listResp.StatusCode()).To(Equal(http.StatusOK)) + ExpectWithOffset(1, listResp.JSON200.Instances).NotTo(BeNil()) + ExpectWithOffset(1, *listResp.JSON200.Instances).To(HaveLen(1), "instance routed to neither agent %q nor %q", agentA, agentB) + + return *(*listResp.JSON200.Instances)[0].Id, agentB } diff --git a/test/subsystem/sp/suite_test.go b/test/subsystem/sp/suite_test.go index 39a343b..8c166e8 100644 --- a/test/subsystem/sp/suite_test.go +++ b/test/subsystem/sp/suite_test.go @@ -3,13 +3,74 @@ package subsystem_test import ( + "context" + "fmt" + "net/http" + "os" "testing" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + agentClient "github.com/dcm-project/control-plane/pkg/agent/client" + catalogClient "github.com/dcm-project/control-plane/pkg/catalog/client" + policyClient "github.com/dcm-project/control-plane/pkg/policy/client" + rmClient "github.com/dcm-project/control-plane/pkg/sp/client/resource_manager" +) + +// Package-level API clients, shared across this suite's test files. All four +// domains (agent, catalog, policy, resource_manager) are served by the same +// control-plane binary at the same base URL. +var ( + rmApiClient *rmClient.ClientWithResponses + agentApiClient *agentClient.ClientWithResponses + catalogApiClient *catalogClient.ClientWithResponses + policyApiClient *policyClient.ClientWithResponses ) func TestE2E(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "E2E Suite") } + +var _ = BeforeSuite(func() { + baseURL := apiBaseURL() + + authEditor := func(_ context.Context, req *http.Request) error { + req.Header.Set("X-Forwarded-User", "test-admin-sub") + return nil + } + + var err error + rmApiClient, err = rmClient.NewClientWithResponses(baseURL, rmClient.WithRequestEditorFn(authEditor)) + Expect(err).NotTo(HaveOccurred()) + + agentApiClient, err = agentClient.NewClientWithResponses(baseURL, agentClient.WithRequestEditorFn(authEditor)) + Expect(err).NotTo(HaveOccurred()) + + catalogApiClient, err = catalogClient.NewClientWithResponses(baseURL, catalogClient.WithRequestEditorFn(authEditor)) + Expect(err).NotTo(HaveOccurred()) + + policyApiClient, err = policyClient.NewClientWithResponses(baseURL, policyClient.WithRequestEditorFn(authEditor)) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() error { + resp, err := http.Get(baseURL + "/health") + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("health check returned %d", resp.StatusCode) + } + return nil + }).WithTimeout(60 * time.Second).WithPolling(2 * time.Second).Should(Succeed()) +}) + +func apiBaseURL() string { + if url := os.Getenv("API_URL"); url != "" { + return url + } + return "http://localhost:8080/api/v1alpha1" +} diff --git a/test/subsystem/sp/testdata/agent_selecting_policy.rego b/test/subsystem/sp/testdata/agent_selecting_policy.rego new file mode 100644 index 0000000..212337a --- /dev/null +++ b/test/subsystem/sp/testdata/agent_selecting_policy.rego @@ -0,0 +1,8 @@ +package sp_subsystem_test + +default main := {"rejected": false} + +main := {"rejected": false, "selected_agent": "__AGENT_NAME__"} if { + some a in input.available_agents + a.name == "__AGENT_NAME__" +} diff --git a/test/subsystem/sp/testdata/three_agent_selecting_policy.rego b/test/subsystem/sp/testdata/three_agent_selecting_policy.rego new file mode 100644 index 0000000..b542615 --- /dev/null +++ b/test/subsystem/sp/testdata/three_agent_selecting_policy.rego @@ -0,0 +1,27 @@ +package sp_subsystem_test_three_agent + +default main := {"rejected": false} + +main := {"rejected": false, "selected_agent": "__AGENT_A__"} if { + some a in input.available_agents + a.name == "__AGENT_A__" +} + +main := {"rejected": false, "selected_agent": "__AGENT_B__"} if { + not agent_available("__AGENT_A__") + some a in input.available_agents + a.name == "__AGENT_B__" +} + +# Unconditional: available_agents is already pre-filtered by service-type +# capability before Rego runs, so this only ever matches when evaluating a +# "database" resource (the only service type agentC is registered for). +main := {"rejected": false, "selected_agent": "__AGENT_C__"} if { + some a in input.available_agents + a.name == "__AGENT_C__" +} + +agent_available(name) if { + some a in input.available_agents + a.name == name +} diff --git a/test/subsystem/sp/testdata/two_agent_selecting_policy.rego b/test/subsystem/sp/testdata/two_agent_selecting_policy.rego new file mode 100644 index 0000000..84b6344 --- /dev/null +++ b/test/subsystem/sp/testdata/two_agent_selecting_policy.rego @@ -0,0 +1,19 @@ +package sp_subsystem_test_selfheal + +default main := {"rejected": false} + +main := {"rejected": false, "selected_agent": "__AGENT_A__"} if { + some a in input.available_agents + a.name == "__AGENT_A__" +} + +main := {"rejected": false, "selected_agent": "__AGENT_B__"} if { + not agent_available("__AGENT_A__") + some a in input.available_agents + a.name == "__AGENT_B__" +} + +agent_available(name) if { + some a in input.available_agents + a.name == name +} From 3611cfab848f4d11a008fd695de5a483c77f64e3 Mon Sep 17 00:00:00 2001 From: gabriel-farache Date: Mon, 10 Aug 2026 14:57:09 +0200 Subject: [PATCH 2/9] feat(sp): log every agent-request publish and response-CE transition Resource lifecycle events (create/queue/cancel/delete) had audit gaps: Publisher never logged publish outcomes, and several response-consumer and sweep transitions were silent on success, so a resource's history couldn't be reconstructed from logs alone. Add centralized publish logging in Publisher.publish, success-path transition logs across response_consumer.go and sweep.go, and unify resource_id/ce_type log keys to instance_id/event_type for consistent querying. Validated with two rounds of independent multi-model review (Opus, Grok, Gemini, Codex) across audit-completeness and code-quality/ operational-safety angles. Round findings closed: missing fields on failure/stale paths, a data race in the test log-capture harness (confirmed via -race), and incomplete test assertions. Assisted by: Cursor - Sonnet 4.5 Signed-off-by: gabriel-farache Co-authored-by: Cursor --- internal/sp/consumer/response_consumer.go | 42 +++--- .../sp/consumer/response_consumer_test.go | 132 +++++++++++++++++- internal/sp/messaging/publisher.go | 22 ++- internal/sp/messaging/publisher_test.go | 44 ++++++ internal/sp/pending/sweep.go | 6 + internal/sp/pending/sweep_test.go | 52 +++++++ 6 files changed, 274 insertions(+), 24 deletions(-) diff --git a/internal/sp/consumer/response_consumer.go b/internal/sp/consumer/response_consumer.go index 481328a..9ac94d8 100644 --- a/internal/sp/consumer/response_consumer.go +++ b/internal/sp/consumer/response_consumer.go @@ -165,7 +165,7 @@ func (c *ResponseConsumer) handleMessage(msg jetstream.Msg) { // agent_name is required on every response CE payload; never fall back // to "trust it anyway" for a missing value. if data.AgentName == "" { - slog.Error("event missing agent_name, acking to discard", "resource_id", data.ResourceID) + slog.Error("event missing agent_name, acking to discard", "instance_id", data.ResourceID) _ = msg.Ack() return } @@ -202,7 +202,7 @@ func (c *ResponseConsumer) handleMessage(msg jetstream.Msg) { newStatus = model.StatusCancelled fromStatuses = []string{model.StatusQueued} default: - slog.Warn("unknown event type, acking", "type", ce.Type) + slog.Warn("unknown event type, acking", "event_type", ce.Type) _ = msg.Ack() return } @@ -210,7 +210,7 @@ func (c *ResponseConsumer) handleMessage(msg jetstream.Msg) { stiStore := c.store.ServiceTypeInstance() applied, err := stiStore.UpdateStatusFrom(ctx, data.ResourceID, fromStatuses, data.AgentName, newStatus, "") if err != nil { - slog.Error("failed to update status, nacking", "resource_id", data.ResourceID, "error", err) + slog.Error("failed to update status, nacking", "instance_id", data.ResourceID, "event_type", ce.Type, "agent_name", data.AgentName, "error", err) _ = msg.NakWithDelay(5 * time.Second) return } @@ -219,7 +219,9 @@ func (c *ResponseConsumer) handleMessage(msg jetstream.Msg) { // reintroduce a TOCTOU window purely for logging; include agent_name // so operators can cross-reference it against the DB instead. slog.Info("stale or duplicate status event, instance already moved on, agent mismatch, or not found, acking", - "resource_id", data.ResourceID, "event_type", ce.Type, "agent_name", data.AgentName) + "instance_id", data.ResourceID, "event_type", ce.Type, "agent_name", data.AgentName) + } else { + slog.Info("status transition applied", "instance_id", data.ResourceID, "event_type", ce.Type, "agent_name", data.AgentName, "status", newStatus) } _ = msg.Ack() @@ -234,14 +236,15 @@ func (c *ResponseConsumer) handleRequestQueued(ctx context.Context, data eventDa if err := c.store.ServiceTypeInstance().MarkQueued(ctx, data.ResourceID, data.AgentName); err != nil { if errors.Is(err, rmstore.ErrInstanceNotFound) { slog.Warn("instance not found, status mismatch, or agent mismatch for request-queued, acking to discard poison message", - "resource_id", data.ResourceID, "agent_name", data.AgentName) + "instance_id", data.ResourceID, "event_type", messaging.CETypeRequestQueued, "agent_name", data.AgentName) _ = msg.Ack() return } - slog.Error("failed to mark instance queued, nacking", "resource_id", data.ResourceID, "error", err) + slog.Error("failed to mark instance queued, nacking", "instance_id", data.ResourceID, "event_type", messaging.CETypeRequestQueued, "agent_name", data.AgentName, "error", err) _ = msg.NakWithDelay(5 * time.Second) return } + slog.Info("status transition applied", "instance_id", data.ResourceID, "event_type", messaging.CETypeRequestQueued, "agent_name", data.AgentName, "status", model.StatusQueued) _ = msg.Ack() } @@ -258,11 +261,11 @@ func (c *ResponseConsumer) handleDeletionAcknowledged(ctx context.Context, data instance, err := stiStore.Get(ctx, data.ResourceID, true) if err != nil { if errors.Is(err, rmstore.ErrInstanceNotFound) { - slog.Info("deletion-acknowledged: instance already gone, acking", "resource_id", data.ResourceID) + slog.Info("deletion-acknowledged: instance already gone, acking", "instance_id", data.ResourceID, "event_type", messaging.CETypeDeletionAcknowledged, "agent_name", data.AgentName) _ = msg.Ack() return } - slog.Error("deletion-acknowledged: failed to look up instance, nacking", "resource_id", data.ResourceID, "error", err) + slog.Error("deletion-acknowledged: failed to look up instance, nacking", "instance_id", data.ResourceID, "event_type", messaging.CETypeDeletionAcknowledged, "agent_name", data.AgentName, "error", err) _ = msg.NakWithDelay(5 * time.Second) return } @@ -274,12 +277,14 @@ func (c *ResponseConsumer) handleDeletionAcknowledged(ctx context.Context, data // its best-effort MarkForDeletion enrollment also set SCHEDULED. if err := stiStore.HardDeleteFromAgent(ctx, data.ResourceID, data.AgentName); err != nil { if !errors.Is(err, rmstore.ErrInstanceNotFound) { - slog.Error("deletion-acknowledged: failed to hard-delete instance, nacking", "resource_id", data.ResourceID, "error", err) + slog.Error("deletion-acknowledged: failed to hard-delete instance, nacking", "instance_id", data.ResourceID, "event_type", messaging.CETypeDeletionAcknowledged, "agent_name", data.AgentName, "error", err) _ = msg.NakWithDelay(5 * time.Second) return } slog.Info("deletion-acknowledged: instance already gone or agent mismatch, acking", - "resource_id", data.ResourceID, "agent_name", data.AgentName) + "instance_id", data.ResourceID, "event_type", messaging.CETypeDeletionAcknowledged, "agent_name", data.AgentName) + } else { + slog.Info("deletion-acknowledged: instance hard-deleted", "instance_id", data.ResourceID, "event_type", messaging.CETypeDeletionAcknowledged, "agent_name", data.AgentName, "status", "DELETED") } case instance.Status == model.StatusPendingDeletion, instance.DeletionStatus != nil && *instance.DeletionStatus == rmstore.DeletionStatusScheduled: @@ -290,16 +295,18 @@ func (c *ResponseConsumer) handleDeletionAcknowledged(ctx context.Context, data if err := stiStore.MarkDeletionCompleteFromAgent(ctx, data.ResourceID, data.AgentName); err != nil { if errors.Is(err, rmstore.ErrInstanceNotFound) { slog.Info("deletion-acknowledged: instance already gone or agent mismatch, acking", - "resource_id", data.ResourceID, "agent_name", data.AgentName) + "instance_id", data.ResourceID, "event_type", messaging.CETypeDeletionAcknowledged, "agent_name", data.AgentName) } else { - slog.Error("deletion-acknowledged: failed to mark deferred deletion complete, nacking", "resource_id", data.ResourceID, "error", err) + slog.Error("deletion-acknowledged: failed to mark deferred deletion complete, nacking", "instance_id", data.ResourceID, "event_type", messaging.CETypeDeletionAcknowledged, "agent_name", data.AgentName, "error", err) _ = msg.NakWithDelay(5 * time.Second) return } + } else { + slog.Info("deletion-acknowledged: deferred deletion marked complete", "instance_id", data.ResourceID, "event_type", messaging.CETypeDeletionAcknowledged, "agent_name", data.AgentName, "status", "DELETED") } default: slog.Info("deletion-acknowledged: deletion already finalized or instance was never deleting, ignoring stale/duplicate ack", - "resource_id", data.ResourceID, "status", instance.Status, "deletion_status", instance.DeletionStatus) + "instance_id", data.ResourceID, "event_type", messaging.CETypeDeletionAcknowledged, "agent_name", data.AgentName, "status", instance.Status, "deletion_status", instance.DeletionStatus) } _ = msg.Ack() } @@ -318,21 +325,22 @@ func (c *ResponseConsumer) handleCancelRejected(ctx context.Context, data eventD cancellableStatuses := []string{model.StatusQueued, model.StatusCancelled} applied, err := stiStore.UpdateStatusFrom(ctx, data.ResourceID, cancellableStatuses, data.AgentName, model.StatusPendingDeletion, "") if err != nil { - slog.Error("cancel-rejected: failed to update status, nacking", "resource_id", data.ResourceID, "error", err) + slog.Error("cancel-rejected: failed to update status, nacking", "instance_id", data.ResourceID, "event_type", messaging.CETypeCancelRejected, "agent_name", data.AgentName, "error", err) _ = msg.NakWithDelay(5 * time.Second) return } if !applied { slog.Info("cancel-rejected: instance already moved to a terminal state or agent mismatch, skipping redundant delete", - "resource_id", data.ResourceID, "agent_name", data.AgentName) + "instance_id", data.ResourceID, "event_type", messaging.CETypeCancelRejected, "agent_name", data.AgentName) _ = msg.Ack() return } + slog.Info("status transition applied", "instance_id", data.ResourceID, "event_type", messaging.CETypeCancelRejected, "agent_name", data.AgentName, "status", model.StatusPendingDeletion) // Enroll in cleanup's retry/timeout tracking, not just the best-effort // republish below: otherwise a failed republish is never retried. if err := stiStore.MarkForDeletion(ctx, data.ResourceID); err != nil { - slog.Error("cancel-rejected: failed to enroll instance in deletion retry tracking", "resource_id", data.ResourceID, "error", err) + slog.Error("cancel-rejected: failed to enroll instance in deletion retry tracking", "instance_id", data.ResourceID, "event_type", messaging.CETypeCancelRejected, "agent_name", data.AgentName, "error", err) } instance, err := stiStore.Get(ctx, data.ResourceID, true) @@ -344,7 +352,7 @@ func (c *ResponseConsumer) handleCancelRejected(ctx context.Context, data eventD ServiceType: instance.ServiceType, } if pubErr := c.publisher.PublishDelete(ctx, subject, payload); pubErr != nil { - slog.Warn("cancel-rejected: publish delete failed, sweep will retry", "resource_id", data.ResourceID, "error", pubErr) + slog.Warn("cancel-rejected: publish delete failed, sweep will retry", "instance_id", data.ResourceID, "error", pubErr) } } } diff --git a/internal/sp/consumer/response_consumer_test.go b/internal/sp/consumer/response_consumer_test.go index 5d8e4c7..2726564 100644 --- a/internal/sp/consumer/response_consumer_test.go +++ b/internal/sp/consumer/response_consumer_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "log/slog" + "sync" "time" agentmodel "github.com/dcm-project/control-plane/internal/agent/store/model" @@ -134,6 +135,27 @@ var _ = Describe("ResponseConsumer", func() { }, 2*time.Second, 20*time.Millisecond).Should(Equal("provisioning")) }) + It("logs the status transition on a successful creation-acknowledged", func() { + instance := createPendingInstance(ctx, db) + + var buf syncBuffer + prevLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + defer slog.SetDefault(prevLogger) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.creation-acknowledged", instance.ID, testAgentName) + + Eventually(buf.String, 2*time.Second, 20*time.Millisecond).Should(SatisfyAll( + ContainSubstring("status transition applied"), + ContainSubstring("instance_id="+instance.ID), + ContainSubstring("event_type=dcm.agent.creation-acknowledged"), + ContainSubstring("agent_name="+testAgentName), + ContainSubstring("status=provisioning"), + )) + }) + // A late creation-acknowledged from a superseded agent must not apply // even though the instance is still "pending" - a status a genuine ack // from the new agent could also legitimately arrive during. @@ -154,7 +176,7 @@ var _ = Describe("ResponseConsumer", func() { It("includes agent_name in the log line when a creation-acknowledged is rejected for an agent mismatch", func() { instance := createPendingInstance(ctx, db) - var buf bytes.Buffer + var buf syncBuffer prevLogger := slog.Default() slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) defer slog.SetDefault(prevLogger) @@ -216,6 +238,27 @@ var _ = Describe("ResponseConsumer", func() { Expect(*updated.PendingStartedAt).To(BeTemporally(">", staleTime)) }) + It("logs the status transition on a successful request-queued", func() { + instance := createPendingInstance(ctx, db) + + var buf syncBuffer + prevLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + defer slog.SetDefault(prevLogger) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.request-queued", instance.ID, testAgentName) + + Eventually(buf.String, 2*time.Second, 20*time.Millisecond).Should(SatisfyAll( + ContainSubstring("status transition applied"), + ContainSubstring("instance_id="+instance.ID), + ContainSubstring("event_type=dcm.agent.request-queued"), + ContainSubstring("agent_name="+testAgentName), + ContainSubstring("status=queued"), + )) + }) + // A stale request-queued from a superseded agent must not mark the // instance queued or reset its pending timer. It("ignores a request-queued from a superseded agent even though status still matches (identity check)", func() { @@ -250,6 +293,28 @@ var _ = Describe("ResponseConsumer", func() { }, 2*time.Second, 100*time.Millisecond).Should(MatchError(gorm.ErrRecordNotFound)) }) + It("logs the hard-delete with event_type on a successful deletion-acknowledged (non-deferred)", func() { + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("status", "deleting").Error).NotTo(HaveOccurred()) + + var buf syncBuffer + prevLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + defer slog.SetDefault(prevLogger) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.deletion-acknowledged", instance.ID, testAgentName) + + Eventually(buf.String, 2*time.Second, 20*time.Millisecond).Should(SatisfyAll( + ContainSubstring("instance hard-deleted"), + ContainSubstring("instance_id="+instance.ID), + ContainSubstring("event_type=dcm.agent.deletion-acknowledged"), + ContainSubstring("agent_name="+testAgentName), + ContainSubstring("status=DELETED"), + )) + }) + // A stale deletion-acknowledged from a superseded agent must not // hard-delete the row, for the non-deferred branch. It("ignores a deletion-acknowledged (non-deferred branch) from a superseded agent", func() { @@ -293,6 +358,28 @@ var _ = Describe("ResponseConsumer", func() { Expect(db.First(&model.ServiceTypeInstance{}, "id = ?", instance.ID).Error).NotTo(HaveOccurred()) }) + It("logs the soft-complete with event_type on a successful deferred deletion-acknowledged", func() { + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("deletion_status", "SCHEDULED").Error).NotTo(HaveOccurred()) + + var buf syncBuffer + prevLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + defer slog.SetDefault(prevLogger) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.deletion-acknowledged", instance.ID, testAgentName) + + Eventually(buf.String, 2*time.Second, 20*time.Millisecond).Should(SatisfyAll( + ContainSubstring("deferred deletion marked complete"), + ContainSubstring("instance_id="+instance.ID), + ContainSubstring("event_type=dcm.agent.deletion-acknowledged"), + ContainSubstring("agent_name="+testAgentName), + ContainSubstring("status=DELETED"), + )) + }) + // Same mismatch treatment, for the deferred (soft-complete) branch. It("ignores a deletion-acknowledged (deferred branch) from a superseded agent", func() { instance := createPendingInstance(ctx, db) @@ -408,6 +495,28 @@ var _ = Describe("ResponseConsumer", func() { Expect(*updated.DeletionStatus).To(Equal("SCHEDULED")) }) + It("logs the status transition on a successful cancel-rejected", func() { + instance := createPendingInstance(ctx, db) + Expect(db.Model(&instance).Update("status", model.StatusQueued).Error).NotTo(HaveOccurred()) + + var buf syncBuffer + prevLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + defer slog.SetDefault(prevLogger) + + Expect(rc.Start(ctx)).To(Succeed()) + + publishAgentEvent(js, "dcm.agent.cancel-rejected", instance.ID, testAgentName) + + Eventually(buf.String, 2*time.Second, 20*time.Millisecond).Should(SatisfyAll( + ContainSubstring("status transition applied"), + ContainSubstring("instance_id="+instance.ID), + ContainSubstring("event_type=dcm.agent.cancel-rejected"), + ContainSubstring("agent_name="+testAgentName), + ContainSubstring("status=pending_deletion"), + )) + }) + It("does not clobber a terminal state on cancel-rejected (CAS guard)", func() { instance := createPendingInstance(ctx, db) Expect(db.Model(&instance).Update("status", "failed").Error).NotTo(HaveOccurred()) @@ -571,6 +680,27 @@ var _ = Describe("ResponseConsumer", func() { }) }) +// syncBuffer wraps bytes.Buffer with a mutex: the consumer writes logs from +// its own background goroutine while these tests concurrently poll the +// buffer's contents via Eventually, which a plain bytes.Buffer doesn't +// support safely. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + // currentStatus polls an instance's current status for use with // Eventually/Consistently, instead of a fixed time.Sleep before one read. func currentStatus(db *gorm.DB, id string) string { diff --git a/internal/sp/messaging/publisher.go b/internal/sp/messaging/publisher.go index 57c0c9a..8960a78 100644 --- a/internal/sp/messaging/publisher.go +++ b/internal/sp/messaging/publisher.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "log/slog" "time" "github.com/cenkalti/backoff/v5" @@ -53,15 +54,15 @@ func (p *Publisher) EnsureStream(ctx context.Context) error { } func (p *Publisher) PublishCreate(ctx context.Context, subject string, payload CreatePayload) error { - return p.publish(ctx, subject, CETypeCreateRequest, payload) + return p.publish(ctx, subject, CETypeCreateRequest, payload.ResourceID, payload) } func (p *Publisher) PublishDelete(ctx context.Context, subject string, payload DeletePayload) error { - return p.publish(ctx, subject, CETypeDeleteRequest, payload) + return p.publish(ctx, subject, CETypeDeleteRequest, payload.ResourceID, payload) } func (p *Publisher) PublishCancel(ctx context.Context, subject string, payload CancelPayload) error { - return p.publish(ctx, subject, CETypeCancelRequest, payload) + return p.publish(ctx, subject, CETypeCancelRequest, payload.ResourceID, payload) } // publish marshals the CloudEvent envelope and publishes it with a bounded @@ -69,8 +70,9 @@ func (p *Publisher) PublishCancel(ctx context.Context, subject string, payload C // header, so a retried publish (by this backoff loop, or by a caller like // the pending sweep re-publishing after a timeout) that reaches JetStream // twice within the dedup window is deduplicated server-side rather than -// producing a duplicate create/delete/cancel request to the agent. -func (p *Publisher) publish(ctx context.Context, subject, ceType string, payload any) error { +// producing a duplicate create/delete/cancel request to the agent. Also +// logs the outcome once, centrally, for every call site. +func (p *Publisher) publish(ctx context.Context, subject, ceType, instanceID string, payload any) error { ceID := uuid.New().String() envelope := map[string]any{ "specversion": CESpecVersion, @@ -80,8 +82,11 @@ func (p *Publisher) publish(ctx context.Context, subject, ceType string, payload "id": ceID, "data": payload, } + log := slog.With("instance_id", instanceID, "subject", subject, "event_type", ceType, "ce_id", ceID) + data, err := json.Marshal(envelope) if err != nil { + log.Error("failed to marshal event, not publishing", "error", err) return err } @@ -90,5 +95,10 @@ func (p *Publisher) publish(ctx context.Context, subject, ceType string, payload return struct{}{}, pubErr } _, err = backoff.Retry(ctx, operation, defaultPublishRetryOptions()...) - return err + if err != nil { + log.Error("event publish failed", "error", err) + return err + } + log.Info("event published") + return nil } diff --git a/internal/sp/messaging/publisher_test.go b/internal/sp/messaging/publisher_test.go index 41b1d04..8b1c5e3 100644 --- a/internal/sp/messaging/publisher_test.go +++ b/internal/sp/messaging/publisher_test.go @@ -1,9 +1,11 @@ package messaging_test import ( + "bytes" "context" "encoding/json" "errors" + "log/slog" "time" "github.com/dcm-project/control-plane/internal/sp/messaging" @@ -242,6 +244,48 @@ var _ = Describe("Publisher", func() { }) }) + Describe("Logging", func() { + It("logs the published event on success with instance_id, subject, event_type and ce_id", func() { + var buf bytes.Buffer + prevLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + defer slog.SetDefault(prevLogger) + + agentTopic := "dcm.agent.prod-eu-west-1" + payload := messaging.CreatePayload{ResourceID: "res-log-1", ServiceType: "vm", Spec: map[string]any{}} + + Expect(publisher.PublishCreate(ctx, agentTopic, payload)).To(Succeed()) + + Expect(buf.String()).To(SatisfyAll( + ContainSubstring("event published"), + ContainSubstring("instance_id=res-log-1"), + ContainSubstring("subject="+agentTopic), + ContainSubstring("event_type=dcm.request.create"), + ContainSubstring("ce_id="), + )) + }) + + It("logs the publish failure once retries are exhausted", func() { + var buf bytes.Buffer + prevLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + defer slog.SetDefault(prevLogger) + + flaky := &flakyJetStream{JetStream: js, failures: 99} + failingPublisher := messaging.NewPublisher(flaky) + + err := failingPublisher.PublishCreate(ctx, "dcm.agent.always-fails", messaging.CreatePayload{ + ResourceID: "res-log-fail", ServiceType: "vm", Spec: map[string]any{}, + }) + Expect(err).To(HaveOccurred()) + + Expect(buf.String()).To(SatisfyAll( + ContainSubstring("event publish failed"), + ContainSubstring("instance_id=res-log-fail"), + )) + }) + }) + Describe("Stream retention", func() { It("EnsureStream configures the agent request stream with WorkQueuePolicy", func() { // BeforeEach already created the stream with default (limits) diff --git a/internal/sp/pending/sweep.go b/internal/sp/pending/sweep.go index f6b43c8..af18997 100644 --- a/internal/sp/pending/sweep.go +++ b/internal/sp/pending/sweep.go @@ -253,6 +253,12 @@ func (s *Sweep) cancelQueuedInstance(ctx context.Context, inst *model.ServiceTyp log.Debug("sweep: queued instance already moved by response consumer or another sweep") return } + agentName := "" + if inst.AgentName != nil { + agentName = *inst.AgentName + } + log.Info("sweep: queued instance timed out waiting for agent acknowledgement, cancelling", + "status", model.StatusCancelled, "agent_name", agentName, "retry_count", inst.RetryCount+1) s.notifyAgentOfCancel(ctx, inst) diff --git a/internal/sp/pending/sweep_test.go b/internal/sp/pending/sweep_test.go index 98c5396..b07fee5 100644 --- a/internal/sp/pending/sweep_test.go +++ b/internal/sp/pending/sweep_test.go @@ -1,8 +1,10 @@ package pending_test import ( + "bytes" "context" "fmt" + "log/slog" "sync" "time" @@ -40,6 +42,27 @@ func limitToSingleConn(d *gorm.DB) { sqlDB.SetMaxOpenConns(1) } +// syncBuffer wraps bytes.Buffer with a mutex: the sweep writes logs from +// its own background goroutine while a test concurrently polls the +// buffer's contents via Eventually, which a plain bytes.Buffer doesn't +// support safely. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + // fakeReevaluator records ReEvaluateWithExclude invocations so tests can // assert the self-healing loop actually calls into placement with the right // arguments, without depending on the real policy/SPRM stack. @@ -292,6 +315,35 @@ var _ = Describe("Queued Sweep", func() { }, time.Second, 10*time.Millisecond).Should(Equal("cancelled")) }) + It("logs the transition when a queued instance times out and is cancelled", func() { + pastTime := time.Now().Add(-2 * time.Minute) + agentName := "test-agent" + instance := model.ServiceTypeInstance{ + ID: uuid.New().String(), + ServiceType: "vm", + Status: "queued", + InstanceName: "queued-sweep-log", + Spec: map[string]any{"cpu": 2}, + AgentName: &agentName, + PendingStartedAt: &pastTime, + } + Expect(db.Create(&instance).Error).NotTo(HaveOccurred()) + + var buf syncBuffer + prevLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, nil))) + defer slog.SetDefault(prevLogger) + + sweep.Start(ctx) + + Eventually(buf.String, time.Second, 10*time.Millisecond).Should(SatisfyAll( + ContainSubstring("timed out waiting for agent acknowledgement"), + ContainSubstring("instance_id="+instance.ID), + ContainSubstring("status=cancelled"), + ContainSubstring("agent_name="+agentName), + )) + }) + It("skips deletion requests", func() { pastTime := time.Now().Add(-2 * time.Minute) agentName := "test-agent" From 0ad5a1ad89c89be1cbb755410417f4c5523c18fd Mon Sep 17 00:00:00 2001 From: Gloria Ciavarrini Date: Thu, 13 Aug 2026 14:26:09 +0200 Subject: [PATCH 3/9] Tmp debug: subsystem black-box jobs now use `@fix/blackbox-timeout-diagnostics` (shared-workflows#32) so a 25m timeout/cancel still dumps podman ps, health, and per-container logs. Main still skips Collect logs on cancel. Will revert to @main once #32 is merged (or when we have enough signal). Not intended to ship. Signed-off-by: Gloria Ciavarrini --- .github/workflows/subsystem.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/subsystem.yaml b/.github/workflows/subsystem.yaml index d5cf120..f530a98 100644 --- a/.github/workflows/subsystem.yaml +++ b/.github/workflows/subsystem.yaml @@ -8,7 +8,7 @@ on: jobs: auth-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@main + uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-timeout-diagnostics with: up-target: auth-subsystem-test-up test-target: auth-subsystem-test @@ -20,7 +20,7 @@ jobs: quay.io/keycloak/keycloak:26.0 policy-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@main + uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-timeout-diagnostics with: up-target: policy-subsystem-test-up test-target: policy-subsystem-test @@ -31,7 +31,7 @@ jobs: quay.io/sclorg/postgresql-16-c9s:latest catalog-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@main + uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-timeout-diagnostics with: up-target: catalog-subsystem-test-up test-target: catalog-subsystem-test @@ -43,7 +43,7 @@ jobs: wiremock/wiremock:3x sp-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@main + uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-timeout-diagnostics with: up-target: sp-subsystem-test-up test-target: sp-subsystem-test From 78fe04ed86479aed560f07ddaf4a7284f78be16f Mon Sep 17 00:00:00 2001 From: Gloria Ciavarrini Date: Fri, 14 Aug 2026 10:31:34 +0200 Subject: [PATCH 4/9] Revert "Tmp debug: subsystem black-box jobs now use `@fix/blackbox-timeout-diagnostics`" This reverts commit 0ad5a1ad89c89be1cbb755410417f4c5523c18fd. --- .github/workflows/subsystem.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/subsystem.yaml b/.github/workflows/subsystem.yaml index f530a98..d5cf120 100644 --- a/.github/workflows/subsystem.yaml +++ b/.github/workflows/subsystem.yaml @@ -8,7 +8,7 @@ on: jobs: auth-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-timeout-diagnostics + uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@main with: up-target: auth-subsystem-test-up test-target: auth-subsystem-test @@ -20,7 +20,7 @@ jobs: quay.io/keycloak/keycloak:26.0 policy-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-timeout-diagnostics + uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@main with: up-target: policy-subsystem-test-up test-target: policy-subsystem-test @@ -31,7 +31,7 @@ jobs: quay.io/sclorg/postgresql-16-c9s:latest catalog-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-timeout-diagnostics + uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@main with: up-target: catalog-subsystem-test-up test-target: catalog-subsystem-test @@ -43,7 +43,7 @@ jobs: wiremock/wiremock:3x sp-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-timeout-diagnostics + uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@main with: up-target: sp-subsystem-test-up test-target: sp-subsystem-test From 1ccdf43505dd918c26ab0be6a0d82baa278a573a Mon Sep 17 00:00:00 2001 From: Gloria Ciavarrini Date: Fri, 14 Aug 2026 10:59:51 +0200 Subject: [PATCH 5/9] do not regress DELETED to FAILED on race MarkDeletionFailed now skips rows already DELETED so a late cleanup timeout cannot overwrite a completed agent ack. Assisted-By: Claude (Anthropic) Signed-off-by: Gloria Ciavarrini --- .../store/resource_manager/service_instance.go | 18 ++++++++++++++---- .../resource_manager/service_instance_test.go | 12 ++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/internal/sp/store/resource_manager/service_instance.go b/internal/sp/store/resource_manager/service_instance.go index b180810..b383b25 100644 --- a/internal/sp/store/resource_manager/service_instance.go +++ b/internal/sp/store/resource_manager/service_instance.go @@ -304,6 +304,7 @@ func (s *ServiceTypeInstanceStore) ExistsByID(ctx context.Context, id string) (b const ( DeletionStatusScheduled = "SCHEDULED" DeletionStatusFailed = "FAILED" + DeletionStatusDeleted = "DELETED" ) func (s *ServiceTypeInstanceStore) MarkForDeletion(ctx context.Context, id string) error { @@ -358,13 +359,22 @@ func (s *ServiceTypeInstanceStore) IncrementDeletionRetry(ctx context.Context, i func (s *ServiceTypeInstanceStore) MarkDeletionFailed(ctx context.Context, id string) error { result := s.db.WithContext(ctx). Model(&model.ServiceTypeInstance{}). - Where("id = ?", id). + Where("id = ? AND deletion_status <> ?", id, DeletionStatusDeleted). Update("deletion_status", DeletionStatusFailed) if result.Error != nil { return result.Error } if result.RowsAffected == 0 { - return ErrInstanceNotFound + var existing model.ServiceTypeInstance + err := s.db.WithContext(ctx).Select("id", "deletion_status").Where("id = ?", id).First(&existing).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return ErrInstanceNotFound + } + if err != nil { + return err + } + // Already DELETED (or otherwise not eligible): do not regress status. + return nil } return nil } @@ -373,7 +383,7 @@ func (s *ServiceTypeInstanceStore) MarkDeletionComplete(ctx context.Context, id result := s.db.WithContext(ctx). Model(&model.ServiceTypeInstance{}). Where("id = ?", id). - Update("deletion_status", "DELETED") + Update("deletion_status", DeletionStatusDeleted) if result.Error != nil { return result.Error } @@ -394,7 +404,7 @@ func (s *ServiceTypeInstanceStore) MarkDeletionCompleteFromAgent(ctx context.Con result := s.db.WithContext(ctx). Model(&model.ServiceTypeInstance{}). Where("id = ? AND agent_name = ?", id, agentName). - Update("deletion_status", "DELETED") + Update("deletion_status", DeletionStatusDeleted) if result.Error != nil { return result.Error } diff --git a/internal/sp/store/resource_manager/service_instance_test.go b/internal/sp/store/resource_manager/service_instance_test.go index a5fa76e..17a86db 100644 --- a/internal/sp/store/resource_manager/service_instance_test.go +++ b/internal/sp/store/resource_manager/service_instance_test.go @@ -723,6 +723,18 @@ var _ = Describe("ServiceTypeInstance Store", func() { err := s.MarkDeletionFailed(ctx, uuid.New().String()) Expect(err).To(MatchError(rmstore.ErrInstanceNotFound)) }) + + It("does not overwrite DELETED with FAILED", func() { + inst := addInstanceToStore(newServiceTypeInstance("fail-after-deleted", map[string]any{})) + Expect(s.MarkForDeletion(ctx, inst.ID)).To(Succeed()) + Expect(s.MarkDeletionComplete(ctx, inst.ID)).To(Succeed()) + + Expect(s.MarkDeletionFailed(ctx, inst.ID)).To(Succeed()) + + found, err := s.Get(ctx, inst.ID, true) + Expect(err).NotTo(HaveOccurred()) + Expect(*found.DeletionStatus).To(Equal("DELETED")) + }) }) Describe("ResetRetryCount", func() { From a546782706fd540da27837aea5368324936eda66 Mon Sep 17 00:00:00 2001 From: Gloria Ciavarrini Date: Fri, 14 Aug 2026 11:01:58 +0200 Subject: [PATCH 6/9] map duplicate instance create to conflict Unique-constraint hits on Create are no longer retried and surface as 409 instead of a generic 500 after backoff. Assisted-By: Claude (Anthropic) Signed-off-by: Gloria Ciavarrini --- .../resource_manager/service_type_instance.go | 3 +++ .../sp/store/resource_manager/service_instance.go | 14 ++++++++++++++ .../resource_manager/service_instance_test.go | 9 +++++++++ 3 files changed, 26 insertions(+) diff --git a/internal/sp/service/resource_manager/service_type_instance.go b/internal/sp/service/resource_manager/service_type_instance.go index b417d80..356efb0 100644 --- a/internal/sp/service/resource_manager/service_type_instance.go +++ b/internal/sp/service/resource_manager/service_type_instance.go @@ -77,6 +77,9 @@ func (s *InstanceService) CreateInstance(ctx context.Context, request *resource_ created, err := s.store.ServiceTypeInstance().Create(ctx, instance) if err != nil { + if errors.Is(err, rmstore.ErrInstanceConflict) { + return nil, service.NewConflictError(fmt.Sprintf("instance with ID '%s' already exists", *instanceID)) + } log.Error("Failed to create instance in store", "instance_id", *instanceID, "error", err) return nil, service.NewInternalError(fmt.Sprintf("failed to create database record for instance %s: %v", *instanceID, err)) } diff --git a/internal/sp/store/resource_manager/service_instance.go b/internal/sp/store/resource_manager/service_instance.go index b383b25..fcc6718 100644 --- a/internal/sp/store/resource_manager/service_instance.go +++ b/internal/sp/store/resource_manager/service_instance.go @@ -22,8 +22,19 @@ var ( // "pending_deletion"), so a self-heal/reassignment can't resurrect it // into "pending" out from under an in-flight delete. ErrInstanceNotEligible = errors.New("instance is not eligible for reassignment") + // ErrInstanceConflict is returned when Create hits a unique-constraint + // violation (typically a duplicate primary key). + ErrInstanceConflict = errors.New("service type instance already exists") ) +func isUniqueViolation(err error) bool { + if errors.Is(err, gorm.ErrDuplicatedKey) { + return true + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "unique constraint") || strings.Contains(msg, "duplicate key") +} + // ServiceTypeInstanceListOptions contains optional fields for listing instances. type ServiceTypeInstanceListOptions struct { ServiceType *string @@ -154,6 +165,9 @@ func (s *ServiceTypeInstanceStore) List(ctx context.Context, opts *ServiceTypeIn func (s *ServiceTypeInstanceStore) Create(ctx context.Context, instance model.ServiceTypeInstance) (*model.ServiceTypeInstance, error) { operation := func() (*model.ServiceTypeInstance, error) { if err := s.db.WithContext(ctx).Clauses(clause.Returning{}).Create(&instance).Error; err != nil { + if isUniqueViolation(err) { + return nil, backoff.Permanent(ErrInstanceConflict) + } return nil, err } return &instance, nil diff --git a/internal/sp/store/resource_manager/service_instance_test.go b/internal/sp/store/resource_manager/service_instance_test.go index 17a86db..f4a0a17 100644 --- a/internal/sp/store/resource_manager/service_instance_test.go +++ b/internal/sp/store/resource_manager/service_instance_test.go @@ -94,6 +94,15 @@ var _ = Describe("ServiceTypeInstance Store", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("database is closed")) }) + + It("returns ErrInstanceConflict for duplicate ID without retrying forever", func() { + instance := newServiceTypeInstance("dup-inst", map[string]any{"cpu": 1}) + _, err := s.Create(ctx, instance) + Expect(err).NotTo(HaveOccurred()) + + _, err = s.Create(ctx, instance) + Expect(err).To(MatchError(rmstore.ErrInstanceConflict)) + }) }) Describe("Get", func() { From d73109e37722a81cc3e32c1d6b47775c0467465c Mon Sep 17 00:00:00 2001 From: Gloria Ciavarrini Date: Fri, 14 Aug 2026 11:31:53 +0200 Subject: [PATCH 7/9] restore instance service debug logs Put back entry/list Debug and resolve-ID Warn/Error logs dropped in the agent rewrite. Assisted-By: Claude (Anthropic) Signed-off-by: Gloria Ciavarrini --- .../resource_manager/service_type_instance.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/sp/service/resource_manager/service_type_instance.go b/internal/sp/service/resource_manager/service_type_instance.go index 356efb0..777c5ee 100644 --- a/internal/sp/service/resource_manager/service_type_instance.go +++ b/internal/sp/service/resource_manager/service_type_instance.go @@ -38,6 +38,7 @@ func NewInstanceService(store store.Store, publisher *messaging.Publisher, agent func (s *InstanceService) CreateInstance(ctx context.Context, request *resource_manager.ServiceTypeInstance, queryID *string, agentName string) (*resource_manager.ServiceTypeInstance, error) { log := logging.FromContext(ctx) + log.Debug("Creating instance", "agent_name", agentName) serviceType, ok := request.Spec["service_type"].(string) if !ok { @@ -217,6 +218,13 @@ func (s *InstanceService) GetInstance(ctx context.Context, instanceID string, sh func (s *InstanceService) ListInstances(ctx context.Context, serviceType, agentName *string, showDeleted bool, maxPageSize *int, pageToken *string) (*resource_manager.ServiceTypeInstanceList, error) { log := logging.FromContext(ctx) + log.Debug("Listing instances", + "service_type", serviceType, + "agent_name", agentName, + "show_deleted", showDeleted, + "max_page_size", maxPageSize, + "has_page_token", pageToken != nil && *pageToken != "", + ) opts := &rmstore.ServiceTypeInstanceListOptions{ ServiceType: serviceType, @@ -247,6 +255,10 @@ func (s *InstanceService) ListInstances(ctx context.Context, serviceType, agentN apiInstances[i] = *ModelToAPI(&inst) } + log.Debug("Instances listed", + "count", len(apiInstances), + "has_next_page", result.NextPageToken != nil, + ) return &resource_manager.ServiceTypeInstanceList{ Instances: &apiInstances, NextPageToken: result.NextPageToken, @@ -263,12 +275,14 @@ func (s *InstanceService) ListInstances(ctx context.Context, serviceType, agentN // tracking the cleanup scheduler uses for deferred deletes. func (s *InstanceService) DeleteInstance(ctx context.Context, instanceID string, deferred bool) error { log := logging.FromContext(ctx) + log.Debug("Deleting instance", "instance_id", instanceID, "deferred", deferred) instance, err := s.store.ServiceTypeInstance().Get(ctx, instanceID, true) if err != nil { if errors.Is(err, rmstore.ErrInstanceNotFound) { return service.NewNotFoundError(fmt.Sprintf("instance %s not found", instanceID)) } + log.Error("Failed to get instance for deletion", "instance_id", instanceID, "error", err) return service.NewInternalError(fmt.Sprintf("failed to retrieve instance: %v", err)) } @@ -370,9 +384,11 @@ func (s *InstanceService) resolveInstanceID(ctx context.Context, queryID *string exists, err := s.store.ServiceTypeInstance().ExistsByID(ctx, requestedID) if err != nil { + log.Error("Failed to check instance ID existence", "instance_id", requestedID, "error", err) return nil, service.NewInternalError(fmt.Sprintf("failed to check instance existence: %v", err)) } if exists { + log.Warn("Duplicate instance ID", "instance_id", requestedID) return nil, service.NewConflictError(fmt.Sprintf("instance with ID '%s' already exists", requestedID)) } From 40337acfcc3981a682c79f73adfa6fb6625a3a39 Mon Sep 17 00:00:00 2001 From: Gloria Ciavarrini Date: Fri, 14 Aug 2026 15:52:50 +0200 Subject: [PATCH 8/9] debug: tmp pin subsystem black-box to shared-workflows#33 Temporary. Revert to @main after that PR merges. Assisted-By: Claude (Anthropic) Signed-off-by: Gloria Ciavarrini --- .github/workflows/subsystem.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/subsystem.yaml b/.github/workflows/subsystem.yaml index d5cf120..1f392de 100644 --- a/.github/workflows/subsystem.yaml +++ b/.github/workflows/subsystem.yaml @@ -6,9 +6,10 @@ on: pull_request: branches: [main, 'release/v*'] +# Temporary pin to shared-workflows#33 — revert to @main after that merges. jobs: auth-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@main + uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-docker-compose-gha with: up-target: auth-subsystem-test-up test-target: auth-subsystem-test @@ -20,7 +21,7 @@ jobs: quay.io/keycloak/keycloak:26.0 policy-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@main + uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-docker-compose-gha with: up-target: policy-subsystem-test-up test-target: policy-subsystem-test @@ -31,7 +32,7 @@ jobs: quay.io/sclorg/postgresql-16-c9s:latest catalog-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@main + uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-docker-compose-gha with: up-target: catalog-subsystem-test-up test-target: catalog-subsystem-test @@ -43,7 +44,7 @@ jobs: wiremock/wiremock:3x sp-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@main + uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-docker-compose-gha with: up-target: sp-subsystem-test-up test-target: sp-subsystem-test From df9c63300e51f2c762d6ff5872c5a3089259b8a9 Mon Sep 17 00:00:00 2001 From: Gloria Ciavarrini Date: Fri, 14 Aug 2026 16:02:04 +0200 Subject: [PATCH 9/9] debug: temp update back-box pin version Signed-off-by: Gloria Ciavarrini --- .github/workflows/subsystem.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/subsystem.yaml b/.github/workflows/subsystem.yaml index 1f392de..200b55f 100644 --- a/.github/workflows/subsystem.yaml +++ b/.github/workflows/subsystem.yaml @@ -6,10 +6,10 @@ on: pull_request: branches: [main, 'release/v*'] -# Temporary pin to shared-workflows#33 — revert to @main after that merges. +# Temporary pin to shared-workflows#33 (fork) — revert to @main after that merges. jobs: auth-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-docker-compose-gha + uses: gciavarrini/dcm-shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-docker-compose-gha with: up-target: auth-subsystem-test-up test-target: auth-subsystem-test @@ -21,7 +21,7 @@ jobs: quay.io/keycloak/keycloak:26.0 policy-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-docker-compose-gha + uses: gciavarrini/dcm-shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-docker-compose-gha with: up-target: policy-subsystem-test-up test-target: policy-subsystem-test @@ -32,7 +32,7 @@ jobs: quay.io/sclorg/postgresql-16-c9s:latest catalog-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-docker-compose-gha + uses: gciavarrini/dcm-shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-docker-compose-gha with: up-target: catalog-subsystem-test-up test-target: catalog-subsystem-test @@ -44,7 +44,7 @@ jobs: wiremock/wiremock:3x sp-subsystem: - uses: dcm-project/shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-docker-compose-gha + uses: gciavarrini/dcm-shared-workflows/.github/workflows/black-box.yaml@fix/blackbox-docker-compose-gha with: up-target: sp-subsystem-test-up test-target: sp-subsystem-test