feat: Add authenticated database readiness check based on driver session metadata

CONFIGURATION.md

@@ -54,6 +54,36 @@ Other Quarkus properties that are specifically relevant for the service:
 | `stargate.jsonapi.operations.database-config.ddl-delay-millis`          | `int`     | `2000`   | Delay between create table and create index to get the schema sync.                                                                                                                                |
 | `stargate.jsonapi.operations.vectorize-enabled`                         | `boolean` | `false`  | Flag to enable server side vectorization.                                                                                                                                              |
 
+### Database readiness
+
+`GET /v1/health/ready` is an authenticated readiness endpoint, the same probe for Astra and
+Cassandra. It uses the request's tenant, `Token`, and `User-Agent` to get a session from the normal
+session cache, and reports ready when that session's driver metadata has a node in the `UP` state.
+No query is issued - getting the session is itself the check. `UP` means this pod holds a working
+session, nothing more; it says nothing about quorum, write availability, or other tenants.
+
+| Status | Body | When |
+|--------|------|------|
+| 200 | `{"status":"UP"}` | session metadata has an `UP` node |
+| 401 | Data API error | `Token` missing or authentication failed |
+| 403 | empty | User-Agent does not match the configured SLA User-Agent |
+| 503 | `{"status":"DOWN"}` | session failure, 5s timeout, or SLA User-Agent not configured |
+
+Probes must treat the HTTP status as the contract, not the body's `status` field.
+
+- **User-Agent** - callers must send the full `stargate.jsonapi.operations.sla-user-agent` value so
+  the probe session gets the shorter SLA cache TTL; anything else is rejected before the session
+  cache is touched, and the endpoint fails closed when that setting is unset. Astra callers must
+  use the probe database hostname so tenant and region resolve from `Host`.
+- **Probe token needs schema read** - `CqlSessionFactory` rejects any new session missing the
+  `system` keyspace, so a token that authenticates but cannot read the schema reports `DOWN` rather
+  than an auth error. Readiness drives pod rotation, so a fleet-wide `DOWN` means a credential
+  problem, not an outage.
+- **Deployment** - probe each pod directly and restrict the endpoint to trusted traffic; the
+  User-Agent check is an operational guard, not an authentication boundary. Kubernetes `httpGet`
+  headers cannot reference a Secret, so deliver the probe token outside Data API config - an
+  external checker, or a Secret-mounted file read by an `exec` probe.
+
 
 ## Jsonapi metering configuration
 *Configuration for jsonapi metering, defined by [JsonApiMetricsConfig.java](io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java).*

src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java

@@ -0,0 +1,69 @@
+package io.stargate.sgv2.jsonapi.api.health;
+
+import com.datastax.oss.driver.api.core.metadata.NodeState;
+import com.google.common.annotations.VisibleForTesting;
+import io.smallrye.mutiny.Uni;
+import io.stargate.sgv2.jsonapi.api.request.RequestContext;
+import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache;
+import java.time.Duration;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+/**
+ * Database probe behind {@code GET /v1/health/ready}, constructed by the JAX-RS resource rather
+ * than as a CDI bean or MicroProfile health check. Succeeds when the session from {@link
+ * CQLSessionCache} has a {@link NodeState#UP} node; no query is issued because an unreachable
+ * database already fails session creation.
+ */
+public final class DatabaseReadinessCheck {
+
+  private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(5);
+
+  private final Supplier<CQLSessionCache> sessionCacheSupplier;
+  private final Duration timeout;
+
+  public DatabaseReadinessCheck(Supplier<CQLSessionCache> sessionCacheSupplier) {
+    this(sessionCacheSupplier, DEFAULT_TIMEOUT);
+  }
+
+  @VisibleForTesting
+  DatabaseReadinessCheck(CQLSessionCache sessionCache, Duration timeout) {
+    this(() -> sessionCache, timeout);
+  }
+
+  private DatabaseReadinessCheck(Supplier<CQLSessionCache> sessionCacheSupplier, Duration timeout) {
+    this.sessionCacheSupplier =
+        Objects.requireNonNull(sessionCacheSupplier, "sessionCacheSupplier must not be null");
+    this.timeout = Objects.requireNonNull(timeout, "timeout must not be null");
+  }
+
+  /** Fails unless the session metadata has at least one {@link NodeState#UP} node. */
+  public Uni<Void> check(RequestContext requestContext) {
+    Objects.requireNonNull(requestContext, "requestContext must not be null");
+
+    return Uni.createFrom()
+        .deferred(
+            () -> {
+              var sessionCache =
+                  Objects.requireNonNull(
+                      sessionCacheSupplier.get(), "sessionCacheSupplier returned null");
+              return sessionCache
+                  .getSession(requestContext)
+                  .invoke(
+                      session -> {
+                        var anyNodeUp =
+                            session.getMetadata().getNodes().values().stream()
+                                .anyMatch(node -> node.getState() == NodeState.UP);
+                        if (!anyNodeUp) {
+                          throw new IllegalStateException(
+                              "Session metadata has no node in the UP state");
+                        }
+                      })
+                  .replaceWithVoid();
+            })
+        // bounds session acquisition; the metadata check itself is in-memory
+        .ifNoItem()
+        .after(timeout)
+        .fail();
+  }
+}

src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java

@@ -0,0 +1,137 @@
+package io.stargate.sgv2.jsonapi.api.v1;
+
+import io.quarkus.security.UnauthorizedException;
+import io.smallrye.mutiny.Uni;
+import io.stargate.sgv2.jsonapi.api.health.DatabaseReadinessCheck;
+import io.stargate.sgv2.jsonapi.api.model.command.CommandResult;
+import io.stargate.sgv2.jsonapi.api.model.command.tracing.RequestTracing;
+import io.stargate.sgv2.jsonapi.api.request.RequestContext;
+import io.stargate.sgv2.jsonapi.config.constants.OpenApiConstants;
+import io.stargate.sgv2.jsonapi.exception.APISecurityException;
+import io.stargate.sgv2.jsonapi.service.cqldriver.CqlSessionCacheSupplier;
+import jakarta.inject.Inject;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import org.eclipse.microprofile.openapi.annotations.Operation;
+import org.eclipse.microprofile.openapi.annotations.media.Content;
+import org.eclipse.microprofile.openapi.annotations.media.Schema;
+import org.eclipse.microprofile.openapi.annotations.responses.APIResponse;
+import org.eclipse.microprofile.openapi.annotations.responses.APIResponses;
+import org.eclipse.microprofile.openapi.annotations.security.SecurityRequirement;
+import org.jboss.resteasy.reactive.RestResponse;
+
+/**
+ * Authenticated readiness endpoint at {@code GET /v1/health/ready}, the same probe for Astra and
+ * Cassandra, see {@link DatabaseReadinessCheck}. The {@code /v1/*} security policy rejects requests
+ * without a token, and callers must send the configured SLA User-Agent so probe sessions get the
+ * shorter SLA cache TTL.
+ */
+@Path(DatabaseReadinessResource.BASE_PATH)
+@Produces(MediaType.APPLICATION_JSON)
+@SecurityRequirement(name = OpenApiConstants.SecuritySchemes.TOKEN)
+public class DatabaseReadinessResource {
+
+  public static final String BASE_PATH = GeneralResource.BASE_PATH + "/health/ready";
+
+  private static final ReadinessResponse UP = new ReadinessResponse("UP");
+  private static final ReadinessResponse DOWN = new ReadinessResponse("DOWN");
+
+  private final DatabaseReadinessCheck readinessCheck;
+  private final RequestContext requestContext;
+  private final CqlSessionCacheSupplier sessionCacheSupplier;
+
+  @Inject
+  public DatabaseReadinessResource(
+      CqlSessionCacheSupplier sessionCacheSupplier, RequestContext requestContext) {
+    this.readinessCheck = new DatabaseReadinessCheck(sessionCacheSupplier);
+    this.requestContext = requestContext;
+    this.sessionCacheSupplier = sessionCacheSupplier;
+  }
+
+  @GET
+  @Operation(
+      summary = "Check database readiness",
+      description =
+          "Uses the authenticated request tenant and token to obtain a session and checks the"
+              + " driver session metadata for an UP node.")
+  @APIResponses({
+    @APIResponse(
+        responseCode = "200",
+        description = "The session metadata reports at least one UP node.",
+        content =
+            @Content(
+                mediaType = MediaType.APPLICATION_JSON,
+                schema = @Schema(implementation = ReadinessResponse.class))),
+    @APIResponse(
+        responseCode = "401",
+        description = "The token is missing or invalid.",
+        content =
+            @Content(
+                mediaType = MediaType.APPLICATION_JSON,
+                schema = @Schema(implementation = CommandResult.class))),
+    @APIResponse(
+        responseCode = "403",
+        description = "The request does not use the configured SLA User-Agent."),
+    @APIResponse(
+        responseCode = "503",
+        description = "The SLA User-Agent is not configured, or the database check failed.",
+        content =
+            @Content(
+                mediaType = MediaType.APPLICATION_JSON,
+                schema = @Schema(implementation = ReadinessResponse.class)))
+  })
+  public Uni<RestResponse<Object>> ready() {
+    var configuredSlaUserAgent = sessionCacheSupplier.slaUserAgent();
+    if (configuredSlaUserAgent.isEmpty()) {
+      return Uni.createFrom().item(response(Response.Status.SERVICE_UNAVAILABLE, DOWN));
+    }
+    if (!configuredSlaUserAgent.get().equals(requestContext.userAgent())) {
+      return Uni.createFrom().item(response(Response.Status.FORBIDDEN));
+    }
+
+    return readinessCheck
+        .check(requestContext)
+        .map(ignored -> response(Response.Status.OK, UP))
+        .onFailure(DatabaseReadinessResource::isUnauthorized)
+        .recoverWithItem(failure -> unauthorizedResponse())
+        .onFailure()
+        .recoverWithItem(failure -> response(Response.Status.SERVICE_UNAVAILABLE, DOWN));
+  }
+
+  private static boolean isUnauthorized(Throwable failure) {
+    var current = failure;
+    var seen = Collections.newSetFromMap(new IdentityHashMap<Throwable, Boolean>());
+    while (current != null && seen.add(current)) {
+      if (current instanceof UnauthorizedException
+          || current instanceof APISecurityException apiException
+              && apiException.httpStatus == Response.Status.UNAUTHORIZED.getStatusCode()) {
+        return true;
+      }
+      current = current.getCause();
+    }
+    return false;
+  }
+
+  private static RestResponse<Object> unauthorizedResponse() {
+    var commandResult =
+        CommandResult.statusOnlyBuilder(RequestTracing.NO_OP)
+            .addThrowable(APISecurityException.Code.UNAUTHENTICATED_REQUEST.get())
+            .build();
+    return response(Response.Status.UNAUTHORIZED, commandResult);
+  }
+
+  private static RestResponse<Object> response(Response.Status status) {
+    return RestResponse.ResponseBuilder.<Object>create(status).build();
+  }
+
+  private static RestResponse<Object> response(Response.Status status, Object entity) {
+    return RestResponse.ResponseBuilder.<Object>create(status, entity).build();
+  }
+
+  public record ReadinessResponse(String status) {}
+}

src/main/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilter.java

@@ -21,6 +21,7 @@
 import io.micrometer.core.instrument.Tag;
 import io.micrometer.core.instrument.Tags;
 import io.stargate.sgv2.jsonapi.api.request.RequestContext;
+import io.stargate.sgv2.jsonapi.api.v1.DatabaseReadinessResource;
 import io.stargate.sgv2.jsonapi.api.v1.metrics.MetricsConfig;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
@@ -71,31 +72,37 @@ public TenantRequestMetricsFilter(
   @ServerResponseFilter
   public void record(
       ContainerRequestContext requestContext, ContainerResponseContext responseContext) {
-    // only if enabled
-    if (config.enabled()) {
-
-      // resolve tenant
-      Tag tenantTag = Tag.of(config.tenantTag(), this.requestContext.tenant().toString());
+    if (!config.enabled() || isDatabaseReadinessRequest(requestContext)) {
+      return;
+    }
 
-      // resolve error
-      boolean error = responseContext.getStatus() >= 500;
-      Tag errorTag = error ? ExceptionMetrics.TAG_ERROR_TRUE : ExceptionMetrics.TAG_ERROR_FALSE;
+    // resolve tenant
+    Tag tenantTag = Tag.of(config.tenantTag(), this.requestContext.tenant().toString());
 
-      // check if we need user agent as well
-      Tags tags = Tags.of(tenantTag, errorTag);
-      if (config.userAgentTagEnabled()) {
-        String userAgentValue = getUserAgentValue(requestContext);
-        tags = tags.and(Tag.of(config.userAgentTag(), userAgentValue));
-      }
+    // resolve error
+    boolean error = responseContext.getStatus() >= 500;
+    Tag errorTag = error ? ExceptionMetrics.TAG_ERROR_TRUE : ExceptionMetrics.TAG_ERROR_FALSE;
 
-      // add http status code
-      if (config.statusTagEnabled()) {
-        tags = tags.and(Tag.of(config.statusTag(), String.valueOf(responseContext.getStatus())));
-      }
+    // check if we need user agent as well
+    Tags tags = Tags.of(tenantTag, errorTag);
+    if (config.userAgentTagEnabled()) {
+      String userAgentValue = getUserAgentValue(requestContext);
+      tags = tags.and(Tag.of(config.userAgentTag(), userAgentValue));
+    }
 
-      // record
-      meterRegistry.counter(config.metricName(), tags).increment();
+    // add http status code
+    if (config.statusTagEnabled()) {
+      tags = tags.and(Tag.of(config.statusTag(), String.valueOf(responseContext.getStatus())));
     }
+
+    // record
+    meterRegistry.counter(config.metricName(), tags).increment();
+  }
+
+  private static boolean isDatabaseReadinessRequest(ContainerRequestContext requestContext) {
+    var requestPath = requestContext.getUriInfo().getRequestUri().getPath();
+    return DatabaseReadinessResource.BASE_PATH.equals(requestPath)
+        || (DatabaseReadinessResource.BASE_PATH + "/").equals(requestPath);
   }
 
   private String getUserAgentValue(ContainerRequestContext requestContext) {

src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionCacheSupplier.java

@@ -10,6 +10,7 @@
 import java.time.Duration;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.function.Supplier;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 
@@ -23,6 +24,7 @@
 public class CqlSessionCacheSupplier implements Supplier<CQLSessionCache> {
 
   private final CQLSessionCache singleton;
+  private final Optional<UserAgent> slaUserAgent;
 
   @Inject
   public CqlSessionCacheSupplier(
@@ -53,11 +55,14 @@ public CqlSessionCacheSupplier(
             dbConfig.cassandraPort(),
             () -> schemaObjectCacheSupplier.get().getSchemaChangeListener());
 
+    slaUserAgent =
+        operationsConfig.slaUserAgent().filter(value -> !value.isBlank()).map(UserAgent::new);
+
     singleton =
         new CQLSessionCache(
             dbConfig.sessionCacheMaxSize(),
             Duration.ofSeconds(dbConfig.sessionCacheTtlSeconds()),
-            operationsConfig.slaUserAgent().map(UserAgent::new).orElse(null),
+            slaUserAgent.orElse(null),
             Duration.ofSeconds(dbConfig.slaSessionCacheTtlSeconds()),
             credentialsFactory,
             sessionFactory,
@@ -70,4 +75,9 @@ public CqlSessionCacheSupplier(
   public CQLSessionCache get() {
     return singleton;
   }
+
+  /** Gets the configured User-Agent that selects the shorter SLA session-cache TTL. */
+  public Optional<UserAgent> slaUserAgent() {
+    return slaUserAgent;
+  }
 }

src/main/resources/application.yaml

@@ -159,7 +159,7 @@ quarkus:
       http-server:
         # ignore all non-application uris, as well as the custom set
         suppress-non-application-uris: true
-        ignore-patterns: /,/metrics,/swagger-ui.*,.*\.html
+        ignore-patterns: /,/metrics,/swagger-ui.*,.*\.html,/v1/health/ready/?
 
         # due to the https://github.com/quarkusio/quarkus/issues/24938
         # we need to define uri templating on our own for now