From d13dda903f2994070ab2a0551932c1eff9739962 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Wed, 12 Aug 2026 18:57:58 +1200 Subject: [PATCH 01/12] FIX: #2539 check CqlSession has metadata Add check when creating a new CqlSession that it has Metadata so we know what keyspaces/tables in the DB. See comments in CqlSessionFactory --- .../jsonapi/exception/DatabaseException.java | 1 + .../service/cqldriver/CqlSessionFactory.java | 41 ++++++- src/main/resources/errors.yaml | 12 +++ .../cqldriver/CqlSessionFactoryTests.java | 101 +++++++++++++++++- .../DefaultDriverExceptionHandlerTest.java | 2 +- 5 files changed, 149 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/exception/DatabaseException.java b/src/main/java/io/stargate/sgv2/jsonapi/exception/DatabaseException.java index dcbfbee125..0c66cb4788 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/exception/DatabaseException.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/exception/DatabaseException.java @@ -18,6 +18,7 @@ public enum Code implements ErrorCode { FAILED_CONCURRENT_OPERATIONS, FAILED_READ_REQUEST, FAILED_TO_CONNECT_TO_DATABASE, + FAILED_TO_READ_METADATA, FAILED_TRUNCATION, FAILED_WRITE_REQUEST, INVALID_DATABASE_QUERY, diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactory.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactory.java index 40cb76cbd0..2a0a610d35 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactory.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactory.java @@ -12,13 +12,17 @@ import com.typesafe.config.ConfigRenderOptions; import io.stargate.sgv2.jsonapi.api.request.tenant.Tenant; import io.stargate.sgv2.jsonapi.config.DatabaseType; +import io.stargate.sgv2.jsonapi.exception.DatabaseException; +import io.stargate.sgv2.jsonapi.exception.ExceptionFlags; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.optvector.SubtypeOnlyFloatVectorToArrayCodec; import io.stargate.sgv2.jsonapi.service.operation.databases.DatabaseDriverExceptionHandler; import io.stargate.sgv2.jsonapi.service.schema.DatabaseSchemaObject; import java.net.InetSocketAddress; import java.util.Collection; +import java.util.EnumSet; import java.util.List; import java.util.Objects; +import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; import java.util.function.Supplier; import org.slf4j.Logger; @@ -213,11 +217,38 @@ public CompletionStage apply(Tenant tenant, CqlCredentials credentia return builder .buildAsync() - .exceptionallyCompose( - throwable -> { - // this will be CompletionException, not the actual cause - throw new DatabaseDriverExceptionHandler(new DatabaseSchemaObject(tenant)) - .maybeHandle(throwable.getCause()); + .handle( + (cqlSession, throwable) -> { + if (throwable != null) { + // there was an error starting the session, often a token is invalid + // When the driver throws it's error passes through CompletionStage's and so is + // wrapped + // in CompletionException. + // Sanity check - only get cause in special case above + var toHandle = + (throwable instanceof CompletionException) ? throwable.getCause() : throwable; + throw new DatabaseDriverExceptionHandler(new DatabaseSchemaObject(tenant)) + .maybeHandle(toHandle); + } + + // Driver auto reads the metadata, unless disabled, and the API must have metadata + // so we can check the keyspace + collection/table exist. + // The metadata read can fail if the token / user+password somehow cannot read the + // schema tables + // or if some coordinators do and some do not validate the credentials. + // In Java Driver see CassandraSchemaQueries.executeOnAdminExecutor() and + // DefaultSession.initialSchemaRefresh() + // NOTE: while throwing the error should prevent the session from getting into the + // CqlSessionCache + // using ExceptionFlags.UNRELIABLE_DB_SESSION tells the CommandProcessor we want to + // evict the session + // when from cache when the request is over. + if (cqlSession.getMetadata().getKeyspace("system").isEmpty()) { + throw DatabaseException.Code.FAILED_TO_READ_METADATA.get( + EnumSet.of(ExceptionFlags.UNRELIABLE_DB_SESSION)); + } + + return cqlSession; }); } } diff --git a/src/main/resources/errors.yaml b/src/main/resources/errors.yaml index 6d267f0457..eb80537c91 100644 --- a/src/main/resources/errors.yaml +++ b/src/main/resources/errors.yaml @@ -2461,6 +2461,18 @@ server-errors: ${SNIPPET.RETRY} + - scope: DATABASE + code: FAILED_TO_READ_METADATA + title: Data API Failed to read database metadata + body: |- + The Data API was unable to read database metadata. + + Database metadata describes the schema including keyspaces, collections, and tables. A connection was established to the database however the driver was unable to read the expected metadata. The API cannot correctly operate without the metadata. + + This may be due to a temporary capacity issue with the database, an issue with token permissions, or a wider outage. + + ${SNIPPET.RETRY} + - scope: DATABASE code: INVALID_COLLECTION_QUERY title: Invalid query diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactoryTests.java b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactoryTests.java index c6b871a43e..ad3d1559ed 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactoryTests.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactoryTests.java @@ -1,19 +1,33 @@ package io.stargate.sgv2.jsonapi.service.cqldriver; +import static io.stargate.sgv2.jsonapi.service.cqldriver.executor.DefaultDriverExceptionHandlerTest.mockNode; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.*; +import com.datastax.oss.driver.api.core.AllNodesFailedException; +import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.CqlSessionBuilder; +import com.datastax.oss.driver.api.core.auth.AuthenticationException; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfigLoader; +import com.datastax.oss.driver.api.core.metadata.Metadata; +import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; import com.datastax.oss.driver.api.core.metadata.schema.SchemaChangeListener; +import com.datastax.oss.driver.internal.core.metadata.DefaultEndPoint; import io.stargate.sgv2.jsonapi.TestConstants; import io.stargate.sgv2.jsonapi.api.request.tenant.Tenant; +import io.stargate.sgv2.jsonapi.exception.APISecurityException; +import io.stargate.sgv2.jsonapi.exception.DatabaseException; +import io.stargate.sgv2.jsonapi.exception.ExceptionFlags; import io.stargate.sgv2.jsonapi.service.cqldriver.executor.optvector.SubtypeOnlyFloatVectorToArrayCodec; import java.net.InetSocketAddress; +import java.util.AbstractMap; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -35,6 +49,64 @@ public void createAstraDbSession() { assertions(fixture, endpoints, schemaListener); } + @Test + public void createAstraDbSessionAuthenticationError() { + + var schemaListener = mock(SchemaChangeListener.class); + var endpoints = List.of(); + + var localHost = InetSocketAddress.createUnresolved("localhost", 9042); + var authErr = + new AuthenticationException( + new DefaultEndPoint(localHost), "A FAKE authentication error occurred"); + + var node = mockNode("node: " + localHost); + var allNodesErr = + AllNodesFailedException.fromErrors(List.of(new AbstractMap.SimpleEntry<>(node, authErr))); + + var fixture = newFixture(TEST_CONSTANTS.TENANT, endpoints, schemaListener, allNodesErr, true); + + // because the CqlSessionFactory is working through java CompletionStage the exception + // the cause is smuggled out in CompletionException + assertThatThrownBy(() -> assertions(fixture, endpoints, schemaListener)) + .as("Authentication error from driver mapped") + .isInstanceOfSatisfying( + CompletionException.class, + completionException -> { + assertThat(completionException.getCause()).isInstanceOf(APISecurityException.class); + APISecurityException err = (APISecurityException) completionException.getCause(); + assertThat(err.code) + .isEqualTo(APISecurityException.Code.UNAUTHENTICATED_REQUEST.name()); + // this is one of the few situations where we return non HTTP 200 + assertThat(err.httpStatus).as("Authentication error is HTTP 401").isEqualTo(401); + }); + } + + @Test + public void createAstraDbSessionMissingMetadata() { + + var schemaListener = mock(SchemaChangeListener.class); + var endpoints = List.of(); + + var fixture = newFixture(TEST_CONSTANTS.TENANT, endpoints, schemaListener, null, false); + + // because the CqlSessionFactory is working through java CompletionStage the exception + // the cause is smuggled out in CompletionException + assertThatThrownBy(() -> assertions(fixture, endpoints, schemaListener)) + .as("Authentication error from driver mapped") + .isInstanceOfSatisfying( + CompletionException.class, + completionException -> { + assertThat(completionException.getCause()).isInstanceOf(DatabaseException.class); + DatabaseException err = (DatabaseException) completionException.getCause(); + assertThat(err.code).isEqualTo(DatabaseException.Code.FAILED_TO_READ_METADATA.name()); + // this is one of the few situations where we return non HTTP 200 + assertThat(err.exceptionFlags) + .as("Exception flagged as UNRELIABLE_DB_SESSION") + .contains(ExceptionFlags.UNRELIABLE_DB_SESSION); + }); + } + @Test public void createCassandraDbSession() { @@ -104,11 +176,29 @@ record Fixture( private Fixture newFixture( Tenant tenant, List endpoints, SchemaChangeListener schemaChangeListener) { + return newFixture(tenant, endpoints, schemaChangeListener, null, true); + } + + private Fixture newFixture( + Tenant tenant, + List endpoints, + SchemaChangeListener schemaChangeListener, + RuntimeException error, + boolean withMetadata) { // we are testing that the CqlSessionFactory calls the session builder correctly, // so we mock the session builder and verify that it is called correctly. var session = mock(CqlSession.class); + // CqlSession guarantees a Metdata obj, and we now check it + var metadata = mock(Metadata.class); + when(session.getMetadata()).thenReturn(metadata); + + Optional keyspaceMetadata = + withMetadata ? Optional.of(mock(KeyspaceMetadata.class)) : Optional.empty(); + when(metadata.getKeyspace(any(CqlIdentifier.class))).thenReturn(keyspaceMetadata); + when(metadata.getKeyspace(anyString())).thenReturn(keyspaceMetadata); + var sessionBuilder = mock(CqlSessionBuilder.class); when(sessionBuilder.withLocalDatacenter(any())).thenReturn(sessionBuilder); when(sessionBuilder.withClassLoader(any())).thenReturn(sessionBuilder); @@ -119,8 +209,15 @@ private Fixture newFixture( when(sessionBuilder.addContactPoints(any())).thenReturn(sessionBuilder); when(sessionBuilder.addTypeCodecs(any())).thenReturn(sessionBuilder); - when(sessionBuilder.buildAsync()).thenReturn(CompletableFuture.completedFuture(session)); - + if (error != null) { + // when the driver completes any error is generates is wrapped in + // CompletionException() because the thown error has passed through stages. + // wrapping here to make it the same + when(sessionBuilder.buildAsync()) + .thenReturn(CompletableFuture.failedFuture(new CompletionException(error))); + } else { + when(sessionBuilder.buildAsync()).thenReturn(CompletableFuture.completedFuture(session)); + } var credentials = mock(CqlCredentials.class); when(credentials.addToSessionBuilder(any())).thenReturn(sessionBuilder); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTest.java b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTest.java index 4286452f1d..8e394502c3 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/DefaultDriverExceptionHandlerTest.java @@ -470,7 +470,7 @@ private static AllNodesFailedException allFailedClusterNodeRecycled() { * *

The errors only use toString (that we have seen so far) */ - private static Node mockNode(String message) { + public static Node mockNode(String message) { return new Node() { @Override public String toString() { From 2a9025953a3e45eb4a41291e7b28d3c9da7fd504 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 13 Aug 2026 11:33:49 +1200 Subject: [PATCH 02/12] bug fix - close session if invalid due to metadata missing --- .../service/cqldriver/CQLSessionCache.java | 1 + .../service/cqldriver/CqlSessionFactory.java | 95 +++++++++++++------ .../cqldriver/CqlSessionFactoryTests.java | 50 +++++++++- 3 files changed, 112 insertions(+), 34 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CQLSessionCache.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CQLSessionCache.java index 3acd54ad65..2fada67edd 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CQLSessionCache.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CQLSessionCache.java @@ -380,6 +380,7 @@ public interface CredentialsFactory extends Function { @FunctionalInterface public interface SessionFactory extends BiFunction> { + CompletionStage apply(Tenant tenant, CqlCredentials credentials); } } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactory.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactory.java index 2a0a610d35..2e3dfcfd4f 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactory.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactory.java @@ -22,8 +22,11 @@ import java.util.EnumSet; import java.util.List; import java.util.Objects; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; +import java.util.function.BiFunction; +import java.util.function.Function; import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -215,40 +218,74 @@ public CompletionStage apply(Tenant tenant, CqlCredentials credentia // Add optimized CqlVector codec (see [data-api#1775]) builder = builder.addTypeCodecs(SubtypeOnlyFloatVectorToArrayCodec.instance()); + // when we are handling the result of the buildAsync() we need the tenant passed along. + // simple recompose the functions so they match signature from the framework + BiFunction partialUnwrapBuildException = + (session, throwable) -> unwrapBuildException(tenant, session, throwable); + Function> partialValidateSession = + (session) -> validateSession(tenant, session); + return builder .buildAsync() - .handle( - (cqlSession, throwable) -> { - if (throwable != null) { - // there was an error starting the session, often a token is invalid - // When the driver throws it's error passes through CompletionStage's and so is - // wrapped - // in CompletionException. - // Sanity check - only get cause in special case above - var toHandle = - (throwable instanceof CompletionException) ? throwable.getCause() : throwable; - throw new DatabaseDriverExceptionHandler(new DatabaseSchemaObject(tenant)) - .maybeHandle(toHandle); - } + .handle(partialUnwrapBuildException) + .thenCompose(partialValidateSession); + } + + /** + * Process any throwable that was thrown from buildAsync(), pass through the session if no error. + */ + private static CqlSession unwrapBuildException( + Tenant tenant, CqlSession cqlSession, Throwable throwable) { + + if (throwable == null) { + return cqlSession; + } - // Driver auto reads the metadata, unless disabled, and the API must have metadata - // so we can check the keyspace + collection/table exist. - // The metadata read can fail if the token / user+password somehow cannot read the - // schema tables - // or if some coordinators do and some do not validate the credentials. - // In Java Driver see CassandraSchemaQueries.executeOnAdminExecutor() and - // DefaultSession.initialSchemaRefresh() - // NOTE: while throwing the error should prevent the session from getting into the - // CqlSessionCache - // using ExceptionFlags.UNRELIABLE_DB_SESSION tells the CommandProcessor we want to - // evict the session - // when from cache when the request is over. - if (cqlSession.getMetadata().getKeyspace("system").isEmpty()) { - throw DatabaseException.Code.FAILED_TO_READ_METADATA.get( - EnumSet.of(ExceptionFlags.UNRELIABLE_DB_SESSION)); + // there was an error starting the session, often a token is invalid + // When the driver throws it's error passes through CompletionStage's and so is + // wrapped in CompletionException. + // Sanity check - only get cause in special case above + var toHandle = + (throwable instanceof CompletionException && throwable.getCause() != null) + ? throwable.getCause() + : throwable; + throw new DatabaseDriverExceptionHandler(new DatabaseSchemaObject(tenant)) + .maybeHandle(toHandle); + } + + /** + * Validate that the session returned from the driver has metadata, close the session and error if + * it is missing. Otherwise, session is good to go. + * + *

Background: Driver auto reads the metadata, unless disabled, and the API must have metadata + * so we can check the keyspace + collection/table exist. The metadata read can fail if the token + * / user+password somehow cannot read the schema tables or if some coordinators do and some do + * not validate the credentials. In Java Driver see + * CassandraSchemaQueries.executeOnAdminExecutor() and DefaultSession.initialSchemaRefresh() + */ + private static CompletionStage validateSession(Tenant tenant, CqlSession cqlSession) { + + if (cqlSession.getMetadata().getKeyspace("system").isPresent()) { + return CompletableFuture.completedStage(cqlSession); + } + + // NOTE: while throwing the error will prevent the session from getting into the + // CqlSessionCache we use ExceptionFlags.UNRELIABLE_DB_SESSION tells the CommandProcessor we + // want to evict the session when from cache when the request is over as belt and braces + return cqlSession + .closeAsync() + .handle( + (ignored, closeError) -> { + if (closeError != null) { + LOGGER.error( + "validateSession() - error closing session when metadata not read, tenant={}", + tenant, + closeError); } - return cqlSession; + // this is the real error we want to get back to the user + throw DatabaseException.Code.FAILED_TO_READ_METADATA.get( + EnumSet.of(ExceptionFlags.UNRELIABLE_DB_SESSION)); }); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactoryTests.java b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactoryTests.java index ad3d1559ed..0450534cd8 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactoryTests.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionFactoryTests.java @@ -64,7 +64,8 @@ public void createAstraDbSessionAuthenticationError() { var allNodesErr = AllNodesFailedException.fromErrors(List.of(new AbstractMap.SimpleEntry<>(node, authErr))); - var fixture = newFixture(TEST_CONSTANTS.TENANT, endpoints, schemaListener, allNodesErr, true); + var fixture = + newFixture(TEST_CONSTANTS.TENANT, endpoints, schemaListener, allNodesErr, true, null); // because the CqlSessionFactory is working through java CompletionStage the exception // the cause is smuggled out in CompletionException @@ -88,7 +89,7 @@ public void createAstraDbSessionMissingMetadata() { var schemaListener = mock(SchemaChangeListener.class); var endpoints = List.of(); - var fixture = newFixture(TEST_CONSTANTS.TENANT, endpoints, schemaListener, null, false); + var fixture = newFixture(TEST_CONSTANTS.TENANT, endpoints, schemaListener, null, false, null); // because the CqlSessionFactory is working through java CompletionStage the exception // the cause is smuggled out in CompletionException @@ -105,6 +106,40 @@ public void createAstraDbSessionMissingMetadata() { .as("Exception flagged as UNRELIABLE_DB_SESSION") .contains(ExceptionFlags.UNRELIABLE_DB_SESSION); }); + + // confirming we are correctly closing the session if metadata is missing + verify(fixture.session()).closeAsync(); + } + + @Test + public void createAstraDbSessionMissingMetadataErrorClosing() { + + var schemaListener = mock(SchemaChangeListener.class); + var endpoints = List.of(); + + var closingError = new RuntimeException("FAKE closeAsync() exception"); + var fixture = + newFixture(TEST_CONSTANTS.TENANT, endpoints, schemaListener, null, false, closingError); + + // same asserts as createAstraDbSessionMissingMetadata - but this is checking that even if + // closeAsync() throws + // the returned error to the user is FAILED_TO_READ_METADATA + assertThatThrownBy(() -> assertions(fixture, endpoints, schemaListener)) + .as("Authentication error from driver mapped") + .isInstanceOfSatisfying( + CompletionException.class, + completionException -> { + assertThat(completionException.getCause()).isInstanceOf(DatabaseException.class); + DatabaseException err = (DatabaseException) completionException.getCause(); + assertThat(err.code).isEqualTo(DatabaseException.Code.FAILED_TO_READ_METADATA.name()); + // this is one of the few situations where we return non HTTP 200 + assertThat(err.exceptionFlags) + .as("Exception flagged as UNRELIABLE_DB_SESSION") + .contains(ExceptionFlags.UNRELIABLE_DB_SESSION); + }); + + // confirming we are correctly closing the session if metadata is missing + verify(fixture.session()).closeAsync(); } @Test @@ -176,7 +211,7 @@ record Fixture( private Fixture newFixture( Tenant tenant, List endpoints, SchemaChangeListener schemaChangeListener) { - return newFixture(tenant, endpoints, schemaChangeListener, null, true); + return newFixture(tenant, endpoints, schemaChangeListener, null, true, null); } private Fixture newFixture( @@ -184,7 +219,8 @@ private Fixture newFixture( List endpoints, SchemaChangeListener schemaChangeListener, RuntimeException error, - boolean withMetadata) { + boolean withMetadata, + RuntimeException closingError) { // we are testing that the CqlSessionFactory calls the session builder correctly, // so we mock the session builder and verify that it is called correctly. @@ -193,7 +229,11 @@ private Fixture newFixture( // CqlSession guarantees a Metdata obj, and we now check it var metadata = mock(Metadata.class); when(session.getMetadata()).thenReturn(metadata); - + if (closingError == null) { + when(session.closeAsync()).thenReturn(CompletableFuture.completedFuture(null)); + } else { + when(session.closeAsync()).thenReturn(CompletableFuture.failedFuture(closingError)); + } Optional keyspaceMetadata = withMetadata ? Optional.of(mock(KeyspaceMetadata.class)) : Optional.empty(); when(metadata.getKeyspace(any(CqlIdentifier.class))).thenReturn(keyspaceMetadata); From 7d30d974cd9005b5aec815cc26422d870e64c017 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Mon, 27 Jul 2026 08:28:42 -0700 Subject: [PATCH 03/12] Add Cassandra readiness health check --- .../CassandraConnectionHealthCheck.java | 170 ++++++++++++++ .../CassandraConnectionHealthCheckTest.java | 208 ++++++++++++++++++ .../v1/SessionEvictionIntegrationTest.java | 55 +++++ 3 files changed, 433 insertions(+) create mode 100644 src/main/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheck.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheck.java b/src/main/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheck.java new file mode 100644 index 0000000000..67c1f06e2c --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheck.java @@ -0,0 +1,170 @@ +package io.stargate.sgv2.jsonapi.api.health; + +import com.datastax.oss.driver.api.core.AllNodesFailedException; +import com.datastax.oss.driver.api.core.connection.ClosedConnectionException; +import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import com.google.common.annotations.VisibleForTesting; +import io.stargate.sgv2.jsonapi.api.request.UserAgent; +import io.stargate.sgv2.jsonapi.api.request.tenant.Tenant; +import io.stargate.sgv2.jsonapi.api.request.tenant.TenantFactory; +import io.stargate.sgv2.jsonapi.config.DatabaseType; +import io.stargate.sgv2.jsonapi.config.OperationsConfig; +import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; +import io.stargate.sgv2.jsonapi.service.cqldriver.CqlCredentials; +import io.stargate.sgv2.jsonapi.service.cqldriver.CqlSessionCacheSupplier; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Base64; +import java.util.Objects; +import org.eclipse.microprofile.health.HealthCheck; +import org.eclipse.microprofile.health.HealthCheckResponse; +import org.eclipse.microprofile.health.Readiness; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Health check that verifies Cassandra connectivity for the Data API readiness probe. + * + *

The check obtains the Cassandra session used by normal requests from the session cache, + * verifies that it is open, and executes a lightweight query against {@code system.local}. It is + * only active when the Data API is configured with a {@link DatabaseType#CASSANDRA} backend. + */ +@Readiness +@ApplicationScoped +public class CassandraConnectionHealthCheck implements HealthCheck { + + private static final Logger LOGGER = + LoggerFactory.getLogger(CassandraConnectionHealthCheck.class); + + private static final String HEALTH_CHECK_NAME = "cassandra-connection"; + private static final String HEALTH_CHECK_QUERY = "SELECT release_version FROM system.local"; + private static final Duration HEALTH_CHECK_TIMEOUT = Duration.ofSeconds(5); + private static final UserAgent HEALTH_CHECK_USER_AGENT = new UserAgent("DataAPI-HealthCheck/1.0"); + + private final CQLSessionCache sessionCache; + private final OperationsConfig operationsConfig; + private final Duration timeout; + private final String authToken; + + @Inject + public CassandraConnectionHealthCheck( + CqlSessionCacheSupplier sessionCacheSupplier, OperationsConfig operationsConfig) { + this( + Objects.requireNonNull(sessionCacheSupplier, "sessionCacheSupplier must not be null").get(), + operationsConfig, + HEALTH_CHECK_TIMEOUT); + } + + @VisibleForTesting + CassandraConnectionHealthCheck( + CQLSessionCache sessionCache, OperationsConfig operationsConfig, Duration timeout) { + this.sessionCache = Objects.requireNonNull(sessionCache, "sessionCache must not be null"); + this.operationsConfig = + Objects.requireNonNull(operationsConfig, "operationsConfig must not be null"); + this.timeout = Objects.requireNonNull(timeout, "timeout must not be null"); + this.authToken = + operationsConfig.databaseConfig().type() == DatabaseType.CASSANDRA + ? createAuthToken(operationsConfig.databaseConfig()) + : null; + } + + @Override + public HealthCheckResponse call() { + var responseBuilder = HealthCheckResponse.named(HEALTH_CHECK_NAME); + + if (operationsConfig.databaseConfig().type() != DatabaseType.CASSANDRA) { + return responseBuilder + .up() + .withData( + "reason", + "Cassandra connectivity check is not applicable for database type " + + operationsConfig.databaseConfig().type()) + .build(); + } + + var healthCheckTenant = TenantFactory.instance().create(null); + + try { + var session = + sessionCache + .getSession(healthCheckTenant, authToken, HEALTH_CHECK_USER_AGENT) + .await() + .atMost(timeout); + + if (session.isClosed()) { + LOGGER.warn("Cassandra session is closed during health check"); + evictSession(healthCheckTenant); + return responseBuilder.down().withData("reason", "Session is closed").build(); + } + + var statement = + SimpleStatement.builder(HEALTH_CHECK_QUERY) + .setTimeout(timeout) + .setConsistencyLevel(operationsConfig.queriesConfig().consistency().reads()) + .build(); + + var resultSet = session.execute(statement); + var row = resultSet.one(); + var version = row != null ? row.getString("release_version") : "unknown"; + + LOGGER.trace("Cassandra health check passed, version: {}", version); + + return responseBuilder + .up() + .withData("cassandra_version", version) + .withData("session_name", session.getName()) + .build(); + } catch (Exception e) { + if (isUnreliableSessionFailure(e)) { + evictSession(healthCheckTenant); + } + + LOGGER.error("Cassandra health check failed", e); + return responseBuilder + .down() + .withData("error", e.getClass().getSimpleName()) + .withData("message", e.getMessage() != null ? e.getMessage() : "Unknown error") + .build(); + } + } + + private static String createAuthToken(OperationsConfig.DatabaseConfig databaseConfig) { + return databaseConfig + .fixedToken() + .orElseGet( + () -> + CqlCredentials.USERNAME_PASSWORD_TOKEN_PREFIX + + encode(databaseConfig.userName()) + + ":" + + encode(databaseConfig.password())); + } + + private static String encode(String value) { + return Base64.getEncoder() + .encodeToString( + Objects.requireNonNull(value, "Cassandra credential must not be null") + .getBytes(StandardCharsets.UTF_8)); + } + + private static boolean isUnreliableSessionFailure(Throwable throwable) { + var current = throwable; + while (current != null) { + if (current instanceof AllNodesFailedException + || current instanceof ClosedConnectionException) { + return true; + } + current = current.getCause(); + } + return false; + } + + private void evictSession(Tenant tenant) { + try { + sessionCache.evictSession(tenant, authToken, HEALTH_CHECK_USER_AGENT); + } catch (Exception e) { + LOGGER.warn("Unable to evict the Cassandra session after a failed health check", e); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java new file mode 100644 index 0000000000..b9898ad60d --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java @@ -0,0 +1,208 @@ +package io.stargate.sgv2.jsonapi.api.health; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.DefaultConsistencyLevel; +import com.datastax.oss.driver.api.core.connection.ClosedConnectionException; +import com.datastax.oss.driver.api.core.cql.ResultSet; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import io.smallrye.mutiny.Uni; +import io.stargate.sgv2.jsonapi.api.request.UserAgent; +import io.stargate.sgv2.jsonapi.api.request.tenant.Tenant; +import io.stargate.sgv2.jsonapi.api.request.tenant.TenantFactory; +import io.stargate.sgv2.jsonapi.config.DatabaseType; +import io.stargate.sgv2.jsonapi.config.OperationsConfig; +import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; +import io.stargate.sgv2.jsonapi.service.cqldriver.CqlCredentials; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Base64; +import java.util.Optional; +import org.eclipse.microprofile.health.HealthCheckResponse; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +/** Tests for {@link CassandraConnectionHealthCheck}. */ +public class CassandraConnectionHealthCheckTest { + + private static final String FIXED_TOKEN = "fixed-token"; + private static final String USER_NAME = "test-user"; + private static final String PASSWORD = "test-password"; + + private CQLSessionCache sessionCache; + private OperationsConfig operationsConfig; + private OperationsConfig.DatabaseConfig databaseConfig; + private CqlSession session; + private CassandraConnectionHealthCheck healthCheck; + + @BeforeEach + public void setup() { + TenantFactory.initialize(DatabaseType.CASSANDRA); + + sessionCache = mock(CQLSessionCache.class); + session = mock(CqlSession.class); + + databaseConfig = mock(OperationsConfig.DatabaseConfig.class); + when(databaseConfig.type()).thenReturn(DatabaseType.CASSANDRA); + when(databaseConfig.fixedToken()).thenReturn(Optional.of(FIXED_TOKEN)); + when(databaseConfig.userName()).thenReturn(USER_NAME); + when(databaseConfig.password()).thenReturn(PASSWORD); + + var consistencyConfig = mock(OperationsConfig.QueriesConfig.ConsistencyConfig.class); + when(consistencyConfig.reads()).thenReturn(DefaultConsistencyLevel.LOCAL_QUORUM); + + var queriesConfig = mock(OperationsConfig.QueriesConfig.class); + when(queriesConfig.consistency()).thenReturn(consistencyConfig); + + operationsConfig = mock(OperationsConfig.class); + when(operationsConfig.databaseConfig()).thenReturn(databaseConfig); + when(operationsConfig.queriesConfig()).thenReturn(queriesConfig); + + healthCheck = + new CassandraConnectionHealthCheck(sessionCache, operationsConfig, Duration.ofMillis(100)); + } + + @AfterEach + public void cleanup() { + TenantFactory.reset(); + } + + @Test + public void successfulHealthCheck() { + var resultSet = mock(ResultSet.class); + var row = mock(Row.class); + + when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) + .thenReturn(Uni.createFrom().item(session)); + when(session.isClosed()).thenReturn(false); + when(session.execute(any(SimpleStatement.class))).thenReturn(resultSet); + when(resultSet.one()).thenReturn(row); + when(row.getString("release_version")).thenReturn("6.9.21"); + when(session.getName()).thenReturn("SINGLE-TENANT"); + + var response = healthCheck.call(); + + assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.UP); + assertThat(response.getData()) + .hasValueSatisfying(data -> assertThat(data).containsEntry("cassandra_version", "6.9.21")); + assertThat(response.getData()) + .hasValueSatisfying( + data -> assertThat(data).containsEntry("session_name", "SINGLE-TENANT")); + + var statementCaptor = ArgumentCaptor.forClass(SimpleStatement.class); + verify(session).execute(statementCaptor.capture()); + assertThat(statementCaptor.getValue().getQuery()) + .isEqualTo("SELECT release_version FROM system.local"); + assertThat(statementCaptor.getValue().getTimeout()).isEqualTo(Duration.ofMillis(100)); + assertThat(statementCaptor.getValue().getConsistencyLevel()) + .isEqualTo(DefaultConsistencyLevel.LOCAL_QUORUM); + } + + @Test + public void sessionAcquisitionFailureReportsDown() { + when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) + .thenReturn(Uni.createFrom().failure(new IllegalStateException("Cannot connect"))); + + var response = healthCheck.call(); + + assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); + assertThat(response.getData()) + .hasValueSatisfying( + data -> + assertThat(data) + .containsEntry("error", "IllegalStateException") + .containsEntry("message", "Cannot connect")); + } + + @Test + public void closedSessionReportsDownAndIsEvicted() { + when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) + .thenReturn(Uni.createFrom().item(session)); + when(session.isClosed()).thenReturn(true); + + var response = healthCheck.call(); + + assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); + assertThat(response.getData()) + .hasValueSatisfying(data -> assertThat(data).containsEntry("reason", "Session is closed")); + verify(sessionCache).evictSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class)); + verify(session, never()).execute(any(SimpleStatement.class)); + } + + @Test + public void queryFailureReportsDownAndUnreliableSessionIsEvicted() { + when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) + .thenReturn(Uni.createFrom().item(session)); + when(session.isClosed()).thenReturn(false); + when(session.execute(any(SimpleStatement.class))) + .thenThrow(new ClosedConnectionException("Connection is closed")); + + var response = healthCheck.call(); + + assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); + assertThat(response.getData()) + .hasValueSatisfying( + data -> + assertThat(data) + .containsEntry("error", "ClosedConnectionException") + .containsEntry("message", "Connection is closed")); + verify(sessionCache).evictSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class)); + } + + @Test + public void sessionAcquisitionTimeoutReportsDown() { + when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) + .thenReturn(Uni.createFrom().nothing()); + + var response = healthCheck.call(); + + assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); + verifyNoInteractions(session); + } + + @Test + public void configuredCredentialsAreUsedWhenFixedTokenIsNotSet() { + when(databaseConfig.fixedToken()).thenReturn(Optional.empty()); + healthCheck = + new CassandraConnectionHealthCheck(sessionCache, operationsConfig, Duration.ofMillis(100)); + + when(sessionCache.getSession(any(Tenant.class), any(String.class), any(UserAgent.class))) + .thenReturn(Uni.createFrom().item(session)); + when(session.isClosed()).thenReturn(true); + + healthCheck.call(); + + var expectedToken = + CqlCredentials.USERNAME_PASSWORD_TOKEN_PREFIX + + Base64.getEncoder().encodeToString(USER_NAME.getBytes(StandardCharsets.UTF_8)) + + ":" + + Base64.getEncoder().encodeToString(PASSWORD.getBytes(StandardCharsets.UTF_8)); + verify(sessionCache).getSession(any(Tenant.class), eq(expectedToken), any(UserAgent.class)); + } + + @Test + public void nonCassandraDatabaseDoesNotRunConnectivityCheck() { + when(databaseConfig.type()).thenReturn(DatabaseType.ASTRA); + healthCheck = + new CassandraConnectionHealthCheck(sessionCache, operationsConfig, Duration.ofMillis(100)); + + var response = healthCheck.call(); + + assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.UP); + assertThat(response.getData()) + .hasValueSatisfying( + data -> + assertThat(data) + .containsEntry( + "reason", + "Cassandra connectivity check is not applicable for database type ASTRA")); + verifyNoInteractions(sessionCache); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java index 322dbd3711..663ed19f60 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java @@ -30,6 +30,8 @@ public class SessionEvictionIntegrationTest extends AbstractCollectionIntegratio private static final Logger LOGGER = LoggerFactory.getLogger(SessionEvictionIntegrationTest.class); + private static final String READINESS_PATH = "/stargate/health/ready"; + private static final String LIVENESS_PATH = "/stargate/health/live"; /** * Overridden to ensure we connect to the isolated container created for this test. @@ -69,6 +71,8 @@ public void testSessionEvictionOnAllNodesFailed() { .body("$", responseIsFindSuccess()) .body("data.document._id", is("before_crash")); + waitForReadinessStatus("UP", 30_000); + // 2. Stop the container to simulate DB failure // Use low-level dockerClient to stop the container without triggering Testcontainers' // cleanup/termination logic (which dbContainer.stop() would do). @@ -86,11 +90,15 @@ public void testSessionEvictionOnAllNodesFailed() { .body( "errors[0].errorCode", is(DatabaseException.Code.FAILED_TO_CONNECT_TO_DATABASE.name())); + waitForReadinessStatus("DOWN", 60_000); + given().when().get(LIVENESS_PATH).then().statusCode(200).body("status", is("UP")); + // 4. Restart the container to simulate recovery getDockerClient().startContainerCmd(getContainerId()).exec(); // 5. Wait for the database to become responsive again waitForDbRecovery(); + waitForReadinessStatus("UP", 120_000); // 6. Verify Session Recovery: check the data before crashing // Not to check that cassandra works, but to check that we are running the same container as @@ -225,6 +233,53 @@ private boolean isApiReady() { } } + /** + * Polls the readiness endpoint until both the overall health and Cassandra connectivity check + * have the expected status. + */ + private void waitForReadinessStatus(String expectedStatus, long timeoutMillis) { + var start = System.currentTimeMillis(); + var expectedStatusCode = "UP".equals(expectedStatus) ? 200 : 503; + Response lastResponse = null; + + while (System.currentTimeMillis() - start < timeoutMillis) { + try { + lastResponse = given().when().get(READINESS_PATH); + var jsonPath = lastResponse.jsonPath(); + var overallStatus = jsonPath.getString("status"); + var cassandraStatus = + jsonPath.getString("checks.find { it.name == 'cassandra-connection' }.status"); + + if (lastResponse.statusCode() == expectedStatusCode + && expectedStatus.equals(overallStatus) + && expectedStatus.equals(cassandraStatus)) { + return; + } + } catch (Exception e) { + LOGGER.debug("Readiness endpoint not in expected state yet", e); + } + + try { + Thread.sleep(1000); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + break; + } + } + + var lastResponseDescription = + lastResponse == null + ? "no response" + : "HTTP " + lastResponse.statusCode() + ": " + lastResponse.asString(); + throw new RuntimeException( + "Readiness did not become " + + expectedStatus + + " within " + + timeoutMillis + + " ms. Last response: " + + lastResponseDescription); + } + /** Checks if Cassandra is up and normal by running "nodetool status" inside the container. */ private boolean isCassandraUp(DockerClient dockerClient, String containerId) { try { From 97a3a3e14851fe331ec0c74216571876c772510b Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Mon, 27 Jul 2026 09:54:05 -0700 Subject: [PATCH 04/12] Address Cassandra readiness review feedback --- CONFIGURATION.md | 22 ++++++++++ .../CassandraConnectionHealthCheck.java | 16 +++++-- .../sgv2/jsonapi/config/OperationsConfig.java | 8 ++-- .../CassandraConnectionHealthCheckTest.java | 43 +++++++++++++++++++ .../v1/SessionEvictionIntegrationTest.java | 2 +- 5 files changed, 83 insertions(+), 8 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index b622e15f60..be02d9ba25 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -54,6 +54,28 @@ Other Quarkus properties that are specifically relevant for the service: | `stargate.jsonapi.operations.database-config.ddl-delay-millis` | `int` | `2000` | Delay between create table and create index to get the schema sync. | | `stargate.jsonapi.operations.vectorize-enabled` | `boolean` | `false` | Flag to enable server side vectorization. | +### Cassandra readiness + +When `stargate.jsonapi.operations.database-config.type` is `CASSANDRA`, the +`/stargate/health/ready` response includes a `cassandra-connection` check. The check obtains a +session through the application session cache and executes +`SELECT release_version FROM system.local`. + +If `stargate.jsonapi.operations.database-config.fixed-token` is configured, the readiness check +uses that token. Otherwise it connects with +`stargate.jsonapi.operations.database-config.user-name` and +`stargate.jsonapi.operations.database-config.password`, which both default to `cassandra`. These +credentials must be valid even when API clients supply different per-request credentials. The check +verifies connectivity with the configured default credentials; it does not validate every +request-specific credential. + +Readiness polling counts as session access and intentionally keeps the cached session active while +polling continues. Session acquisition and the validation query each have a five-second +timeout, so deployment probe timeouts should allow for both stages. The Helm chart defaults the +readiness probe timeout to ten seconds. + +For other database types, the check reports UP without accessing the Cassandra session cache. + ## Jsonapi metering configuration *Configuration for jsonapi metering, defined by [JsonApiMetricsConfig.java](io/stargate/sgv2/jsonapi/api/v1/metrics/JsonApiMetricsConfig.java).* diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheck.java b/src/main/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheck.java index 67c1f06e2c..b5d0d2ccbc 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheck.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheck.java @@ -27,9 +27,18 @@ /** * Health check that verifies Cassandra connectivity for the Data API readiness probe. * - *

The check obtains the Cassandra session used by normal requests from the session cache, - * verifies that it is open, and executes a lightweight query against {@code system.local}. It is - * only active when the Data API is configured with a {@link DatabaseType#CASSANDRA} backend. + *

The check obtains a session through the same session cache used by normal requests, verifies + * that it is open, and executes a lightweight query against {@code system.local}. It uses the fixed + * token when configured; otherwise it uses the configured Cassandra username and password. These + * default credentials verify base Cassandra connectivity, not the validity of every credential + * supplied on a request. + * + *

Readiness polling deliberately keeps the cached session active while polling continues. If + * requests use the same credentials, they share that session; otherwise the check maintains a + * dedicated session for the configured default credentials. + * + *

The database type is runtime configuration, so the bean remains registered for other database + * types. In those deployments it reports UP without accessing the Cassandra session cache. */ @Readiness @ApplicationScoped @@ -149,6 +158,7 @@ private static String encode(String value) { } private static boolean isUnreliableSessionFailure(Throwable throwable) { + // Session acquisition through Mutiny may wrap driver failures, so inspect the cause chain. var current = throwable; while (current != null) { if (current instanceof AllNodesFailedException diff --git a/src/main/java/io/stargate/sgv2/jsonapi/config/OperationsConfig.java b/src/main/java/io/stargate/sgv2/jsonapi/config/OperationsConfig.java index 981d89da89..63d56f7024 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/config/OperationsConfig.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/config/OperationsConfig.java @@ -220,16 +220,16 @@ interface DatabaseConfig { DatabaseType type(); /** - * Username when connecting to cassandra database (when type is {@link DatabaseType#CASSANDRA}) - * and fixedToken is used + * Username used for Cassandra connections when fixedToken is configured, and by the Cassandra + * readiness check when fixedToken is not configured. */ @Nullable @WithDefault("cassandra") String userName(); /** - * Password when connecting to cassandra database (when type is {@link DatabaseType#CASSANDRA}) - * and fixedToken is used + * Password used for Cassandra connections when fixedToken is configured, and by the Cassandra + * readiness check when fixedToken is not configured. */ @Nullable @WithDefault("cassandra") diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java index b9898ad60d..582741d7c8 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java @@ -5,6 +5,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.*; +import com.datastax.oss.driver.api.core.AllNodesFailedException; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.DefaultConsistencyLevel; import com.datastax.oss.driver.api.core.connection.ClosedConnectionException; @@ -44,6 +45,7 @@ public class CassandraConnectionHealthCheckTest { @BeforeEach public void setup() { + TenantFactory.reset(); TenantFactory.initialize(DatabaseType.CASSANDRA); sessionCache = mock(CQLSessionCache.class); @@ -119,6 +121,8 @@ public void sessionAcquisitionFailureReportsDown() { assertThat(data) .containsEntry("error", "IllegalStateException") .containsEntry("message", "Cannot connect")); + verify(sessionCache, never()) + .evictSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class)); } @Test @@ -156,6 +160,43 @@ public void queryFailureReportsDownAndUnreliableSessionIsEvicted() { verify(sessionCache).evictSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class)); } + @Test + public void wrappedAllNodesFailedReportsDownAndUnreliableSessionIsEvicted() { + var allNodesFailed = mock(AllNodesFailedException.class); + + when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) + .thenReturn(Uni.createFrom().item(session)); + when(session.isClosed()).thenReturn(false); + when(session.execute(any(SimpleStatement.class))) + .thenThrow(new RuntimeException("Session failed", allNodesFailed)); + + var response = healthCheck.call(); + + assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); + assertThat(response.getData()) + .hasValueSatisfying( + data -> + assertThat(data) + .containsEntry("error", "RuntimeException") + .containsEntry("message", "Session failed")); + verify(sessionCache).evictSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class)); + } + + @Test + public void queryFailureDoesNotEvictReliableSession() { + when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) + .thenReturn(Uni.createFrom().item(session)); + when(session.isClosed()).thenReturn(false); + when(session.execute(any(SimpleStatement.class))) + .thenThrow(new IllegalStateException("Invalid query")); + + var response = healthCheck.call(); + + assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); + verify(sessionCache, never()) + .evictSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class)); + } + @Test public void sessionAcquisitionTimeoutReportsDown() { when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) @@ -164,6 +205,8 @@ public void sessionAcquisitionTimeoutReportsDown() { var response = healthCheck.call(); assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); + verify(sessionCache, never()) + .evictSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class)); verifyNoInteractions(session); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java index 663ed19f60..28ec1c2907 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java @@ -271,7 +271,7 @@ private void waitForReadinessStatus(String expectedStatus, long timeoutMillis) { lastResponse == null ? "no response" : "HTTP " + lastResponse.statusCode() + ": " + lastResponse.asString(); - throw new RuntimeException( + throw new AssertionError( "Readiness did not become " + expectedStatus + " within " From 638a7be57c1ad0f538faf86bf29a923d675a4f9e Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Mon, 27 Jul 2026 10:11:35 -0700 Subject: [PATCH 05/12] Reuse shared Cassandra test fixtures --- .../CassandraConnectionHealthCheckTest.java | 68 +++++++++++-------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java index 582741d7c8..bbdf6e5ef2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java @@ -13,8 +13,8 @@ import com.datastax.oss.driver.api.core.cql.Row; import com.datastax.oss.driver.api.core.cql.SimpleStatement; import io.smallrye.mutiny.Uni; +import io.stargate.sgv2.jsonapi.TestConstants; import io.stargate.sgv2.jsonapi.api.request.UserAgent; -import io.stargate.sgv2.jsonapi.api.request.tenant.Tenant; import io.stargate.sgv2.jsonapi.api.request.tenant.TenantFactory; import io.stargate.sgv2.jsonapi.config.DatabaseType; import io.stargate.sgv2.jsonapi.config.OperationsConfig; @@ -33,10 +33,12 @@ /** Tests for {@link CassandraConnectionHealthCheck}. */ public class CassandraConnectionHealthCheckTest { - private static final String FIXED_TOKEN = "fixed-token"; + private static final UserAgent HEALTH_CHECK_USER_AGENT = new UserAgent("DataAPI-HealthCheck/1.0"); private static final String USER_NAME = "test-user"; private static final String PASSWORD = "test-password"; + private final TestConstants TEST_CONSTANTS = new TestConstants(); + private CQLSessionCache sessionCache; private OperationsConfig operationsConfig; private OperationsConfig.DatabaseConfig databaseConfig; @@ -53,7 +55,7 @@ public void setup() { databaseConfig = mock(OperationsConfig.DatabaseConfig.class); when(databaseConfig.type()).thenReturn(DatabaseType.CASSANDRA); - when(databaseConfig.fixedToken()).thenReturn(Optional.of(FIXED_TOKEN)); + when(databaseConfig.fixedToken()).thenReturn(Optional.of(TEST_CONSTANTS.AUTH_TOKEN)); when(databaseConfig.userName()).thenReturn(USER_NAME); when(databaseConfig.password()).thenReturn(PASSWORD); @@ -81,8 +83,7 @@ public void successfulHealthCheck() { var resultSet = mock(ResultSet.class); var row = mock(Row.class); - when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) - .thenReturn(Uni.createFrom().item(session)); + sessionRequestReturns(Uni.createFrom().item(session)); when(session.isClosed()).thenReturn(false); when(session.execute(any(SimpleStatement.class))).thenReturn(resultSet); when(resultSet.one()).thenReturn(row); @@ -109,8 +110,7 @@ public void successfulHealthCheck() { @Test public void sessionAcquisitionFailureReportsDown() { - when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) - .thenReturn(Uni.createFrom().failure(new IllegalStateException("Cannot connect"))); + sessionRequestReturns(Uni.createFrom().failure(new IllegalStateException("Cannot connect"))); var response = healthCheck.call(); @@ -121,14 +121,12 @@ public void sessionAcquisitionFailureReportsDown() { assertThat(data) .containsEntry("error", "IllegalStateException") .containsEntry("message", "Cannot connect")); - verify(sessionCache, never()) - .evictSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class)); + verifySessionNotEvicted(); } @Test public void closedSessionReportsDownAndIsEvicted() { - when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) - .thenReturn(Uni.createFrom().item(session)); + sessionRequestReturns(Uni.createFrom().item(session)); when(session.isClosed()).thenReturn(true); var response = healthCheck.call(); @@ -136,14 +134,13 @@ public void closedSessionReportsDownAndIsEvicted() { assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); assertThat(response.getData()) .hasValueSatisfying(data -> assertThat(data).containsEntry("reason", "Session is closed")); - verify(sessionCache).evictSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class)); + verifySessionEvicted(); verify(session, never()).execute(any(SimpleStatement.class)); } @Test public void queryFailureReportsDownAndUnreliableSessionIsEvicted() { - when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) - .thenReturn(Uni.createFrom().item(session)); + sessionRequestReturns(Uni.createFrom().item(session)); when(session.isClosed()).thenReturn(false); when(session.execute(any(SimpleStatement.class))) .thenThrow(new ClosedConnectionException("Connection is closed")); @@ -157,15 +154,14 @@ public void queryFailureReportsDownAndUnreliableSessionIsEvicted() { assertThat(data) .containsEntry("error", "ClosedConnectionException") .containsEntry("message", "Connection is closed")); - verify(sessionCache).evictSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class)); + verifySessionEvicted(); } @Test public void wrappedAllNodesFailedReportsDownAndUnreliableSessionIsEvicted() { var allNodesFailed = mock(AllNodesFailedException.class); - when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) - .thenReturn(Uni.createFrom().item(session)); + sessionRequestReturns(Uni.createFrom().item(session)); when(session.isClosed()).thenReturn(false); when(session.execute(any(SimpleStatement.class))) .thenThrow(new RuntimeException("Session failed", allNodesFailed)); @@ -179,13 +175,12 @@ public void wrappedAllNodesFailedReportsDownAndUnreliableSessionIsEvicted() { assertThat(data) .containsEntry("error", "RuntimeException") .containsEntry("message", "Session failed")); - verify(sessionCache).evictSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class)); + verifySessionEvicted(); } @Test public void queryFailureDoesNotEvictReliableSession() { - when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) - .thenReturn(Uni.createFrom().item(session)); + sessionRequestReturns(Uni.createFrom().item(session)); when(session.isClosed()).thenReturn(false); when(session.execute(any(SimpleStatement.class))) .thenThrow(new IllegalStateException("Invalid query")); @@ -193,20 +188,17 @@ public void queryFailureDoesNotEvictReliableSession() { var response = healthCheck.call(); assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); - verify(sessionCache, never()) - .evictSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class)); + verifySessionNotEvicted(); } @Test public void sessionAcquisitionTimeoutReportsDown() { - when(sessionCache.getSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class))) - .thenReturn(Uni.createFrom().nothing()); + sessionRequestReturns(Uni.createFrom().nothing()); var response = healthCheck.call(); assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); - verify(sessionCache, never()) - .evictSession(any(Tenant.class), eq(FIXED_TOKEN), any(UserAgent.class)); + verifySessionNotEvicted(); verifyNoInteractions(session); } @@ -216,7 +208,8 @@ public void configuredCredentialsAreUsedWhenFixedTokenIsNotSet() { healthCheck = new CassandraConnectionHealthCheck(sessionCache, operationsConfig, Duration.ofMillis(100)); - when(sessionCache.getSession(any(Tenant.class), any(String.class), any(UserAgent.class))) + when(sessionCache.getSession( + eq(TEST_CONSTANTS.CASSANDRA_TENANT), any(String.class), eq(HEALTH_CHECK_USER_AGENT))) .thenReturn(Uni.createFrom().item(session)); when(session.isClosed()).thenReturn(true); @@ -227,7 +220,8 @@ public void configuredCredentialsAreUsedWhenFixedTokenIsNotSet() { + Base64.getEncoder().encodeToString(USER_NAME.getBytes(StandardCharsets.UTF_8)) + ":" + Base64.getEncoder().encodeToString(PASSWORD.getBytes(StandardCharsets.UTF_8)); - verify(sessionCache).getSession(any(Tenant.class), eq(expectedToken), any(UserAgent.class)); + verify(sessionCache) + .getSession(TEST_CONSTANTS.CASSANDRA_TENANT, expectedToken, HEALTH_CHECK_USER_AGENT); } @Test @@ -248,4 +242,22 @@ public void nonCassandraDatabaseDoesNotRunConnectivityCheck() { "Cassandra connectivity check is not applicable for database type ASTRA")); verifyNoInteractions(sessionCache); } + + private void sessionRequestReturns(Uni sessionResult) { + when(sessionCache.getSession( + TEST_CONSTANTS.CASSANDRA_TENANT, TEST_CONSTANTS.AUTH_TOKEN, HEALTH_CHECK_USER_AGENT)) + .thenReturn(sessionResult); + } + + private void verifySessionEvicted() { + verify(sessionCache) + .evictSession( + TEST_CONSTANTS.CASSANDRA_TENANT, TEST_CONSTANTS.AUTH_TOKEN, HEALTH_CHECK_USER_AGENT); + } + + private void verifySessionNotEvicted() { + verify(sessionCache, never()) + .evictSession( + TEST_CONSTANTS.CASSANDRA_TENANT, TEST_CONSTANTS.AUTH_TOKEN, HEALTH_CHECK_USER_AGENT); + } } From 8d84c456b77ed9436f8b7e776c1369a92feacbb0 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Mon, 3 Aug 2026 20:20:52 -0700 Subject: [PATCH 06/12] chore: refactor to support astra readiness --- CONFIGURATION.md | 54 ++-- .../CassandraConnectionHealthCheck.java | 180 ------------ .../api/health/DatabaseReadinessCheck.java | 72 +++++ .../api/v1/DatabaseReadinessResource.java | 103 +++++++ .../sgv2/jsonapi/config/OperationsConfig.java | 8 +- .../metrics/TenantRequestMetricsFilter.java | 46 +-- src/main/resources/application.yaml | 2 +- .../CassandraConnectionHealthCheckTest.java | 263 ------------------ .../health/DatabaseReadinessCheckTest.java | 147 ++++++++++ .../api/v1/DatabaseReadinessResourceTest.java | 181 ++++++++++++ .../v1/SessionEvictionIntegrationTest.java | 34 ++- .../TenantRequestMetricsFilterTest.java | 40 +++ 12 files changed, 627 insertions(+), 503 deletions(-) delete mode 100644 src/main/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheck.java create mode 100644 src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java create mode 100644 src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheckTest.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResourceTest.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilterTest.java diff --git a/CONFIGURATION.md b/CONFIGURATION.md index be02d9ba25..70e0b43147 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -54,27 +54,39 @@ Other Quarkus properties that are specifically relevant for the service: | `stargate.jsonapi.operations.database-config.ddl-delay-millis` | `int` | `2000` | Delay between create table and create index to get the schema sync. | | `stargate.jsonapi.operations.vectorize-enabled` | `boolean` | `false` | Flag to enable server side vectorization. | -### Cassandra readiness - -When `stargate.jsonapi.operations.database-config.type` is `CASSANDRA`, the -`/stargate/health/ready` response includes a `cassandra-connection` check. The check obtains a -session through the application session cache and executes -`SELECT release_version FROM system.local`. - -If `stargate.jsonapi.operations.database-config.fixed-token` is configured, the readiness check -uses that token. Otherwise it connects with -`stargate.jsonapi.operations.database-config.user-name` and -`stargate.jsonapi.operations.database-config.password`, which both default to `cassandra`. These -credentials must be valid even when API clients supply different per-request credentials. The check -verifies connectivity with the configured default credentials; it does not validate every -request-specific credential. - -Readiness polling counts as session access and intentionally keeps the cached session active while -polling continues. Session acquisition and the validation query each have a five-second -timeout, so deployment probe timeouts should allow for both stages. The Helm chart defaults the -readiness probe timeout to ten seconds. - -For other database types, the check reports UP without accessing the Cassandra session cache. +### Database readiness + +`GET /v1/health/ready` is an authenticated database readiness endpoint used for both Astra and +Cassandra deployments. It uses the request's tenant, `Token` header, and `User-Agent` to obtain a +session through the normal session cache. The Data API does not store separate readiness +credentials. + +The endpoint executes `SELECT * FROM datastax_sla.check LIMIT 1` at `LOCAL_QUORUM`, using the +`table-read` driver profile for the remaining read settings. An `UP` response therefore confirms +that the coordinator can complete a read from a replicated table at local quorum. It does not +validate every tenant's credentials, write availability, or cross-region availability. + +The deployment must provide a dedicated canary tenant and credentials for this request and must +provision a `datastax_sla.check` table that the canary principal can read. Its replication factor +must be appropriate for the deployment (greater than one in a multi-node local data center) so +`LOCAL_QUORUM` requires responses from multiple replicas. Astra callers must use the canary database +hostname so the tenant and region are resolved from `Host`; Cassandra ignores the tenant portion of +`Host`. The caller must also send the exact User-Agent configured by +`stargate.jsonapi.operations.sla-user-agent`, allowing a dedicated canary session to use the shorter +SLA session TTL instead of being treated like normal client traffic. + +The check is fully asynchronous and has a five-second timeout. It returns HTTP 200 with +`{"status":"UP"}` after a successful read, HTTP 503 with `{"status":"DOWN"}` after a database +failure or timeout, and HTTP 401 when the `Token` header is missing or authentication fails. + +Kubernetes or an SLA checker must call each pod directly for this endpoint to control per-pod +readiness. An external request sent through a load balancer does not establish which pod is ready. +Restrict the endpoint to trusted probe traffic with deployment controls such as a NetworkPolicy, +mTLS, or an ingress ACL and rate limit. Kubernetes `httpGet` headers cannot reference a Secret, so +delivery of the canary token is intentionally outside the Data API configuration. Prefer an +external checker or a Secret-mounted file read by an `exec` probe; do not put the token literally in +the probe command or shell trace. The unauthenticated Quarkus health endpoints under the +non-application path do not include this database check. ## Jsonapi metering configuration diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheck.java b/src/main/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheck.java deleted file mode 100644 index b5d0d2ccbc..0000000000 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheck.java +++ /dev/null @@ -1,180 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.health; - -import com.datastax.oss.driver.api.core.AllNodesFailedException; -import com.datastax.oss.driver.api.core.connection.ClosedConnectionException; -import com.datastax.oss.driver.api.core.cql.SimpleStatement; -import com.google.common.annotations.VisibleForTesting; -import io.stargate.sgv2.jsonapi.api.request.UserAgent; -import io.stargate.sgv2.jsonapi.api.request.tenant.Tenant; -import io.stargate.sgv2.jsonapi.api.request.tenant.TenantFactory; -import io.stargate.sgv2.jsonapi.config.DatabaseType; -import io.stargate.sgv2.jsonapi.config.OperationsConfig; -import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; -import io.stargate.sgv2.jsonapi.service.cqldriver.CqlCredentials; -import io.stargate.sgv2.jsonapi.service.cqldriver.CqlSessionCacheSupplier; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Base64; -import java.util.Objects; -import org.eclipse.microprofile.health.HealthCheck; -import org.eclipse.microprofile.health.HealthCheckResponse; -import org.eclipse.microprofile.health.Readiness; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Health check that verifies Cassandra connectivity for the Data API readiness probe. - * - *

The check obtains a session through the same session cache used by normal requests, verifies - * that it is open, and executes a lightweight query against {@code system.local}. It uses the fixed - * token when configured; otherwise it uses the configured Cassandra username and password. These - * default credentials verify base Cassandra connectivity, not the validity of every credential - * supplied on a request. - * - *

Readiness polling deliberately keeps the cached session active while polling continues. If - * requests use the same credentials, they share that session; otherwise the check maintains a - * dedicated session for the configured default credentials. - * - *

The database type is runtime configuration, so the bean remains registered for other database - * types. In those deployments it reports UP without accessing the Cassandra session cache. - */ -@Readiness -@ApplicationScoped -public class CassandraConnectionHealthCheck implements HealthCheck { - - private static final Logger LOGGER = - LoggerFactory.getLogger(CassandraConnectionHealthCheck.class); - - private static final String HEALTH_CHECK_NAME = "cassandra-connection"; - private static final String HEALTH_CHECK_QUERY = "SELECT release_version FROM system.local"; - private static final Duration HEALTH_CHECK_TIMEOUT = Duration.ofSeconds(5); - private static final UserAgent HEALTH_CHECK_USER_AGENT = new UserAgent("DataAPI-HealthCheck/1.0"); - - private final CQLSessionCache sessionCache; - private final OperationsConfig operationsConfig; - private final Duration timeout; - private final String authToken; - - @Inject - public CassandraConnectionHealthCheck( - CqlSessionCacheSupplier sessionCacheSupplier, OperationsConfig operationsConfig) { - this( - Objects.requireNonNull(sessionCacheSupplier, "sessionCacheSupplier must not be null").get(), - operationsConfig, - HEALTH_CHECK_TIMEOUT); - } - - @VisibleForTesting - CassandraConnectionHealthCheck( - CQLSessionCache sessionCache, OperationsConfig operationsConfig, Duration timeout) { - this.sessionCache = Objects.requireNonNull(sessionCache, "sessionCache must not be null"); - this.operationsConfig = - Objects.requireNonNull(operationsConfig, "operationsConfig must not be null"); - this.timeout = Objects.requireNonNull(timeout, "timeout must not be null"); - this.authToken = - operationsConfig.databaseConfig().type() == DatabaseType.CASSANDRA - ? createAuthToken(operationsConfig.databaseConfig()) - : null; - } - - @Override - public HealthCheckResponse call() { - var responseBuilder = HealthCheckResponse.named(HEALTH_CHECK_NAME); - - if (operationsConfig.databaseConfig().type() != DatabaseType.CASSANDRA) { - return responseBuilder - .up() - .withData( - "reason", - "Cassandra connectivity check is not applicable for database type " - + operationsConfig.databaseConfig().type()) - .build(); - } - - var healthCheckTenant = TenantFactory.instance().create(null); - - try { - var session = - sessionCache - .getSession(healthCheckTenant, authToken, HEALTH_CHECK_USER_AGENT) - .await() - .atMost(timeout); - - if (session.isClosed()) { - LOGGER.warn("Cassandra session is closed during health check"); - evictSession(healthCheckTenant); - return responseBuilder.down().withData("reason", "Session is closed").build(); - } - - var statement = - SimpleStatement.builder(HEALTH_CHECK_QUERY) - .setTimeout(timeout) - .setConsistencyLevel(operationsConfig.queriesConfig().consistency().reads()) - .build(); - - var resultSet = session.execute(statement); - var row = resultSet.one(); - var version = row != null ? row.getString("release_version") : "unknown"; - - LOGGER.trace("Cassandra health check passed, version: {}", version); - - return responseBuilder - .up() - .withData("cassandra_version", version) - .withData("session_name", session.getName()) - .build(); - } catch (Exception e) { - if (isUnreliableSessionFailure(e)) { - evictSession(healthCheckTenant); - } - - LOGGER.error("Cassandra health check failed", e); - return responseBuilder - .down() - .withData("error", e.getClass().getSimpleName()) - .withData("message", e.getMessage() != null ? e.getMessage() : "Unknown error") - .build(); - } - } - - private static String createAuthToken(OperationsConfig.DatabaseConfig databaseConfig) { - return databaseConfig - .fixedToken() - .orElseGet( - () -> - CqlCredentials.USERNAME_PASSWORD_TOKEN_PREFIX - + encode(databaseConfig.userName()) - + ":" - + encode(databaseConfig.password())); - } - - private static String encode(String value) { - return Base64.getEncoder() - .encodeToString( - Objects.requireNonNull(value, "Cassandra credential must not be null") - .getBytes(StandardCharsets.UTF_8)); - } - - private static boolean isUnreliableSessionFailure(Throwable throwable) { - // Session acquisition through Mutiny may wrap driver failures, so inspect the cause chain. - var current = throwable; - while (current != null) { - if (current instanceof AllNodesFailedException - || current instanceof ClosedConnectionException) { - return true; - } - current = current.getCause(); - } - return false; - } - - private void evictSession(Tenant tenant) { - try { - sessionCache.evictSession(tenant, authToken, HEALTH_CHECK_USER_AGENT); - } catch (Exception e) { - LOGGER.warn("Unable to evict the Cassandra session after a failed health check", e); - } - } -} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java b/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java new file mode 100644 index 0000000000..bbec7687d1 --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java @@ -0,0 +1,72 @@ +package io.stargate.sgv2.jsonapi.api.health; + +import com.datastax.oss.driver.api.core.DefaultConsistencyLevel; +import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import com.google.common.annotations.VisibleForTesting; +import io.smallrye.mutiny.Uni; +import io.stargate.sgv2.jsonapi.api.request.RequestContext; +import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; +import io.stargate.sgv2.jsonapi.service.cqldriver.executor.CommandQueryExecutor; +import java.time.Duration; +import java.util.Objects; +import java.util.function.Supplier; + +/** + * Runs the database probe exposed at {@code GET /v1/health/ready}. + * + *

This class is constructed by the JAX-RS resource and is not a CDI bean or a MicroProfile + * health check. The caller's request context supplies the tenant, token, and User-Agent for both + * Astra and Cassandra connections. + */ +public final class DatabaseReadinessCheck { + + private static final String READINESS_QUERY = "SELECT * FROM datastax_sla.check LIMIT 1"; + private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(5); + + private final Supplier sessionCacheSupplier; + private final SimpleStatement statement; + private final Duration timeout; + + public DatabaseReadinessCheck(Supplier sessionCacheSupplier) { + this(sessionCacheSupplier, DEFAULT_TIMEOUT); + } + + @VisibleForTesting + DatabaseReadinessCheck(CQLSessionCache sessionCache, Duration timeout) { + this(() -> sessionCache, timeout); + } + + private DatabaseReadinessCheck(Supplier sessionCacheSupplier, Duration timeout) { + this.sessionCacheSupplier = + Objects.requireNonNull(sessionCacheSupplier, "sessionCacheSupplier must not be null"); + this.timeout = Objects.requireNonNull(timeout, "timeout must not be null"); + this.statement = + SimpleStatement.builder(READINESS_QUERY) + .setConsistencyLevel(DefaultConsistencyLevel.LOCAL_QUORUM) + .setTimeout(timeout) + .build(); + } + + /** + * Executes a replicated table read at {@code LOCAL_QUORUM}, using the {@code table-read} driver + * profile for the remaining read settings. + */ + public Uni check(RequestContext requestContext) { + Objects.requireNonNull(requestContext, "requestContext must not be null"); + + return Uni.createFrom() + .deferred( + () -> { + var sessionCache = + Objects.requireNonNull( + sessionCacheSupplier.get(), "sessionCacheSupplier returned null"); + return new CommandQueryExecutor( + sessionCache, requestContext, CommandQueryExecutor.QueryTarget.TABLE) + .executeRead(statement) + .replaceWithVoid(); + }) + .ifNoItem() + .after(timeout) + .fail(); + } +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java new file mode 100644 index 0000000000..38bafaa95f --- /dev/null +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java @@ -0,0 +1,103 @@ +package io.stargate.sgv2.jsonapi.api.v1; + +import io.quarkus.security.UnauthorizedException; +import io.smallrye.mutiny.Uni; +import io.stargate.sgv2.jsonapi.api.health.DatabaseReadinessCheck; +import io.stargate.sgv2.jsonapi.api.request.RequestContext; +import io.stargate.sgv2.jsonapi.config.constants.OpenApiConstants; +import io.stargate.sgv2.jsonapi.exception.APISecurityException; +import io.stargate.sgv2.jsonapi.service.cqldriver.CqlSessionCacheSupplier; +import jakarta.inject.Inject; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import org.eclipse.microprofile.openapi.annotations.Operation; +import org.eclipse.microprofile.openapi.annotations.media.Content; +import org.eclipse.microprofile.openapi.annotations.media.Schema; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; +import org.eclipse.microprofile.openapi.annotations.responses.APIResponses; +import org.eclipse.microprofile.openapi.annotations.security.SecurityRequirement; +import org.jboss.resteasy.reactive.RestResponse; + +/** + * Authenticated database readiness endpoint registered through Quarkus JAX-RS resource discovery. + * + *

{@code GET /v1/health/ready} runs the same request-scoped probe for Astra and Cassandra. A + * successful probe returns HTTP 200; invalid credentials return HTTP 401; and a database failure or + * timeout returns HTTP 503. The existing {@code /v1/*} security policy rejects requests without a + * token before this resource is called. + */ +@Path(DatabaseReadinessResource.BASE_PATH) +@Produces(MediaType.APPLICATION_JSON) +@SecurityRequirement(name = OpenApiConstants.SecuritySchemes.TOKEN) +public class DatabaseReadinessResource { + + public static final String BASE_PATH = GeneralResource.BASE_PATH + "/health/ready"; + + private static final ReadinessResponse UP = new ReadinessResponse("UP"); + private static final ReadinessResponse DOWN = new ReadinessResponse("DOWN"); + + private final DatabaseReadinessCheck readinessCheck; + private final RequestContext requestContext; + + @Inject + public DatabaseReadinessResource( + CqlSessionCacheSupplier sessionCacheSupplier, RequestContext requestContext) { + this.readinessCheck = new DatabaseReadinessCheck(sessionCacheSupplier); + this.requestContext = requestContext; + } + + @GET + @Operation( + summary = "Check database readiness", + description = + "Uses the authenticated request tenant and token to perform a LOCAL_QUORUM read.") + @APIResponses({ + @APIResponse( + responseCode = "200", + description = "The database completed the readiness read.", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema(implementation = ReadinessResponse.class))), + @APIResponse(responseCode = "401", description = "The token is missing or invalid."), + @APIResponse( + responseCode = "503", + description = "The database read failed or timed out.", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema(implementation = ReadinessResponse.class))) + }) + public Uni> ready() { + return readinessCheck + .check(requestContext) + .map(ignored -> RestResponse.ok(UP)) + .onFailure(DatabaseReadinessResource::isUnauthorized) + .recoverWithItem( + failure -> + RestResponse.ResponseBuilder.create(Response.Status.UNAUTHORIZED, DOWN).build()) + .onFailure() + .recoverWithItem( + failure -> + RestResponse.ResponseBuilder.create(Response.Status.SERVICE_UNAVAILABLE, DOWN) + .build()); + } + + private static boolean isUnauthorized(Throwable failure) { + var current = failure; + while (current != null) { + if (current instanceof UnauthorizedException + || current instanceof APISecurityException apiException + && apiException.httpStatus == Response.Status.UNAUTHORIZED.getStatusCode()) { + return true; + } + current = current.getCause(); + } + return false; + } + + public record ReadinessResponse(String status) {} +} diff --git a/src/main/java/io/stargate/sgv2/jsonapi/config/OperationsConfig.java b/src/main/java/io/stargate/sgv2/jsonapi/config/OperationsConfig.java index 63d56f7024..981d89da89 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/config/OperationsConfig.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/config/OperationsConfig.java @@ -220,16 +220,16 @@ interface DatabaseConfig { DatabaseType type(); /** - * Username used for Cassandra connections when fixedToken is configured, and by the Cassandra - * readiness check when fixedToken is not configured. + * Username when connecting to cassandra database (when type is {@link DatabaseType#CASSANDRA}) + * and fixedToken is used */ @Nullable @WithDefault("cassandra") String userName(); /** - * Password used for Cassandra connections when fixedToken is configured, and by the Cassandra - * readiness check when fixedToken is not configured. + * Password when connecting to cassandra database (when type is {@link DatabaseType#CASSANDRA}) + * and fixedToken is used */ @Nullable @WithDefault("cassandra") diff --git a/src/main/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilter.java b/src/main/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilter.java index e365b4891e..c633208225 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilter.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilter.java @@ -21,6 +21,7 @@ import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Tags; import io.stargate.sgv2.jsonapi.api.request.RequestContext; +import io.stargate.sgv2.jsonapi.api.v1.DatabaseReadinessResource; import io.stargate.sgv2.jsonapi.api.v1.metrics.MetricsConfig; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -71,31 +72,36 @@ public TenantRequestMetricsFilter( @ServerResponseFilter public void record( ContainerRequestContext requestContext, ContainerResponseContext responseContext) { - // only if enabled - if (config.enabled()) { - - // resolve tenant - Tag tenantTag = Tag.of(config.tenantTag(), this.requestContext.tenant().toString()); + if (!config.enabled() || isDatabaseReadinessRequest(requestContext)) { + return; + } - // resolve error - boolean error = responseContext.getStatus() >= 500; - Tag errorTag = error ? ExceptionMetrics.TAG_ERROR_TRUE : ExceptionMetrics.TAG_ERROR_FALSE; + // resolve tenant + Tag tenantTag = Tag.of(config.tenantTag(), this.requestContext.tenant().toString()); - // check if we need user agent as well - Tags tags = Tags.of(tenantTag, errorTag); - if (config.userAgentTagEnabled()) { - String userAgentValue = getUserAgentValue(requestContext); - tags = tags.and(Tag.of(config.userAgentTag(), userAgentValue)); - } + // resolve error + boolean error = responseContext.getStatus() >= 500; + Tag errorTag = error ? ExceptionMetrics.TAG_ERROR_TRUE : ExceptionMetrics.TAG_ERROR_FALSE; - // add http status code - if (config.statusTagEnabled()) { - tags = tags.and(Tag.of(config.statusTag(), String.valueOf(responseContext.getStatus()))); - } + // check if we need user agent as well + Tags tags = Tags.of(tenantTag, errorTag); + if (config.userAgentTagEnabled()) { + String userAgentValue = getUserAgentValue(requestContext); + tags = tags.and(Tag.of(config.userAgentTag(), userAgentValue)); + } - // record - meterRegistry.counter(config.metricName(), tags).increment(); + // add http status code + if (config.statusTagEnabled()) { + tags = tags.and(Tag.of(config.statusTag(), String.valueOf(responseContext.getStatus()))); } + + // record + meterRegistry.counter(config.metricName(), tags).increment(); + } + + private static boolean isDatabaseReadinessRequest(ContainerRequestContext requestContext) { + return DatabaseReadinessResource.BASE_PATH.equals( + requestContext.getUriInfo().getRequestUri().getPath()); } private String getUserAgentValue(ContainerRequestContext requestContext) { diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 354232535c..1103fef1b3 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -159,7 +159,7 @@ quarkus: http-server: # ignore all non-application uris, as well as the custom set suppress-non-application-uris: true - ignore-patterns: /,/metrics,/swagger-ui.*,.*\.html + ignore-patterns: /,/metrics,/swagger-ui.*,.*\.html,/v1/health/ready # due to the https://github.com/quarkusio/quarkus/issues/24938 # we need to define uri templating on our own for now diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java deleted file mode 100644 index bbdf6e5ef2..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/health/CassandraConnectionHealthCheckTest.java +++ /dev/null @@ -1,263 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.health; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.*; - -import com.datastax.oss.driver.api.core.AllNodesFailedException; -import com.datastax.oss.driver.api.core.CqlSession; -import com.datastax.oss.driver.api.core.DefaultConsistencyLevel; -import com.datastax.oss.driver.api.core.connection.ClosedConnectionException; -import com.datastax.oss.driver.api.core.cql.ResultSet; -import com.datastax.oss.driver.api.core.cql.Row; -import com.datastax.oss.driver.api.core.cql.SimpleStatement; -import io.smallrye.mutiny.Uni; -import io.stargate.sgv2.jsonapi.TestConstants; -import io.stargate.sgv2.jsonapi.api.request.UserAgent; -import io.stargate.sgv2.jsonapi.api.request.tenant.TenantFactory; -import io.stargate.sgv2.jsonapi.config.DatabaseType; -import io.stargate.sgv2.jsonapi.config.OperationsConfig; -import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; -import io.stargate.sgv2.jsonapi.service.cqldriver.CqlCredentials; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Base64; -import java.util.Optional; -import org.eclipse.microprofile.health.HealthCheckResponse; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -/** Tests for {@link CassandraConnectionHealthCheck}. */ -public class CassandraConnectionHealthCheckTest { - - private static final UserAgent HEALTH_CHECK_USER_AGENT = new UserAgent("DataAPI-HealthCheck/1.0"); - private static final String USER_NAME = "test-user"; - private static final String PASSWORD = "test-password"; - - private final TestConstants TEST_CONSTANTS = new TestConstants(); - - private CQLSessionCache sessionCache; - private OperationsConfig operationsConfig; - private OperationsConfig.DatabaseConfig databaseConfig; - private CqlSession session; - private CassandraConnectionHealthCheck healthCheck; - - @BeforeEach - public void setup() { - TenantFactory.reset(); - TenantFactory.initialize(DatabaseType.CASSANDRA); - - sessionCache = mock(CQLSessionCache.class); - session = mock(CqlSession.class); - - databaseConfig = mock(OperationsConfig.DatabaseConfig.class); - when(databaseConfig.type()).thenReturn(DatabaseType.CASSANDRA); - when(databaseConfig.fixedToken()).thenReturn(Optional.of(TEST_CONSTANTS.AUTH_TOKEN)); - when(databaseConfig.userName()).thenReturn(USER_NAME); - when(databaseConfig.password()).thenReturn(PASSWORD); - - var consistencyConfig = mock(OperationsConfig.QueriesConfig.ConsistencyConfig.class); - when(consistencyConfig.reads()).thenReturn(DefaultConsistencyLevel.LOCAL_QUORUM); - - var queriesConfig = mock(OperationsConfig.QueriesConfig.class); - when(queriesConfig.consistency()).thenReturn(consistencyConfig); - - operationsConfig = mock(OperationsConfig.class); - when(operationsConfig.databaseConfig()).thenReturn(databaseConfig); - when(operationsConfig.queriesConfig()).thenReturn(queriesConfig); - - healthCheck = - new CassandraConnectionHealthCheck(sessionCache, operationsConfig, Duration.ofMillis(100)); - } - - @AfterEach - public void cleanup() { - TenantFactory.reset(); - } - - @Test - public void successfulHealthCheck() { - var resultSet = mock(ResultSet.class); - var row = mock(Row.class); - - sessionRequestReturns(Uni.createFrom().item(session)); - when(session.isClosed()).thenReturn(false); - when(session.execute(any(SimpleStatement.class))).thenReturn(resultSet); - when(resultSet.one()).thenReturn(row); - when(row.getString("release_version")).thenReturn("6.9.21"); - when(session.getName()).thenReturn("SINGLE-TENANT"); - - var response = healthCheck.call(); - - assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.UP); - assertThat(response.getData()) - .hasValueSatisfying(data -> assertThat(data).containsEntry("cassandra_version", "6.9.21")); - assertThat(response.getData()) - .hasValueSatisfying( - data -> assertThat(data).containsEntry("session_name", "SINGLE-TENANT")); - - var statementCaptor = ArgumentCaptor.forClass(SimpleStatement.class); - verify(session).execute(statementCaptor.capture()); - assertThat(statementCaptor.getValue().getQuery()) - .isEqualTo("SELECT release_version FROM system.local"); - assertThat(statementCaptor.getValue().getTimeout()).isEqualTo(Duration.ofMillis(100)); - assertThat(statementCaptor.getValue().getConsistencyLevel()) - .isEqualTo(DefaultConsistencyLevel.LOCAL_QUORUM); - } - - @Test - public void sessionAcquisitionFailureReportsDown() { - sessionRequestReturns(Uni.createFrom().failure(new IllegalStateException("Cannot connect"))); - - var response = healthCheck.call(); - - assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); - assertThat(response.getData()) - .hasValueSatisfying( - data -> - assertThat(data) - .containsEntry("error", "IllegalStateException") - .containsEntry("message", "Cannot connect")); - verifySessionNotEvicted(); - } - - @Test - public void closedSessionReportsDownAndIsEvicted() { - sessionRequestReturns(Uni.createFrom().item(session)); - when(session.isClosed()).thenReturn(true); - - var response = healthCheck.call(); - - assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); - assertThat(response.getData()) - .hasValueSatisfying(data -> assertThat(data).containsEntry("reason", "Session is closed")); - verifySessionEvicted(); - verify(session, never()).execute(any(SimpleStatement.class)); - } - - @Test - public void queryFailureReportsDownAndUnreliableSessionIsEvicted() { - sessionRequestReturns(Uni.createFrom().item(session)); - when(session.isClosed()).thenReturn(false); - when(session.execute(any(SimpleStatement.class))) - .thenThrow(new ClosedConnectionException("Connection is closed")); - - var response = healthCheck.call(); - - assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); - assertThat(response.getData()) - .hasValueSatisfying( - data -> - assertThat(data) - .containsEntry("error", "ClosedConnectionException") - .containsEntry("message", "Connection is closed")); - verifySessionEvicted(); - } - - @Test - public void wrappedAllNodesFailedReportsDownAndUnreliableSessionIsEvicted() { - var allNodesFailed = mock(AllNodesFailedException.class); - - sessionRequestReturns(Uni.createFrom().item(session)); - when(session.isClosed()).thenReturn(false); - when(session.execute(any(SimpleStatement.class))) - .thenThrow(new RuntimeException("Session failed", allNodesFailed)); - - var response = healthCheck.call(); - - assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); - assertThat(response.getData()) - .hasValueSatisfying( - data -> - assertThat(data) - .containsEntry("error", "RuntimeException") - .containsEntry("message", "Session failed")); - verifySessionEvicted(); - } - - @Test - public void queryFailureDoesNotEvictReliableSession() { - sessionRequestReturns(Uni.createFrom().item(session)); - when(session.isClosed()).thenReturn(false); - when(session.execute(any(SimpleStatement.class))) - .thenThrow(new IllegalStateException("Invalid query")); - - var response = healthCheck.call(); - - assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); - verifySessionNotEvicted(); - } - - @Test - public void sessionAcquisitionTimeoutReportsDown() { - sessionRequestReturns(Uni.createFrom().nothing()); - - var response = healthCheck.call(); - - assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.DOWN); - verifySessionNotEvicted(); - verifyNoInteractions(session); - } - - @Test - public void configuredCredentialsAreUsedWhenFixedTokenIsNotSet() { - when(databaseConfig.fixedToken()).thenReturn(Optional.empty()); - healthCheck = - new CassandraConnectionHealthCheck(sessionCache, operationsConfig, Duration.ofMillis(100)); - - when(sessionCache.getSession( - eq(TEST_CONSTANTS.CASSANDRA_TENANT), any(String.class), eq(HEALTH_CHECK_USER_AGENT))) - .thenReturn(Uni.createFrom().item(session)); - when(session.isClosed()).thenReturn(true); - - healthCheck.call(); - - var expectedToken = - CqlCredentials.USERNAME_PASSWORD_TOKEN_PREFIX - + Base64.getEncoder().encodeToString(USER_NAME.getBytes(StandardCharsets.UTF_8)) - + ":" - + Base64.getEncoder().encodeToString(PASSWORD.getBytes(StandardCharsets.UTF_8)); - verify(sessionCache) - .getSession(TEST_CONSTANTS.CASSANDRA_TENANT, expectedToken, HEALTH_CHECK_USER_AGENT); - } - - @Test - public void nonCassandraDatabaseDoesNotRunConnectivityCheck() { - when(databaseConfig.type()).thenReturn(DatabaseType.ASTRA); - healthCheck = - new CassandraConnectionHealthCheck(sessionCache, operationsConfig, Duration.ofMillis(100)); - - var response = healthCheck.call(); - - assertThat(response.getStatus()).isEqualTo(HealthCheckResponse.Status.UP); - assertThat(response.getData()) - .hasValueSatisfying( - data -> - assertThat(data) - .containsEntry( - "reason", - "Cassandra connectivity check is not applicable for database type ASTRA")); - verifyNoInteractions(sessionCache); - } - - private void sessionRequestReturns(Uni sessionResult) { - when(sessionCache.getSession( - TEST_CONSTANTS.CASSANDRA_TENANT, TEST_CONSTANTS.AUTH_TOKEN, HEALTH_CHECK_USER_AGENT)) - .thenReturn(sessionResult); - } - - private void verifySessionEvicted() { - verify(sessionCache) - .evictSession( - TEST_CONSTANTS.CASSANDRA_TENANT, TEST_CONSTANTS.AUTH_TOKEN, HEALTH_CHECK_USER_AGENT); - } - - private void verifySessionNotEvicted() { - verify(sessionCache, never()) - .evictSession( - TEST_CONSTANTS.CASSANDRA_TENANT, TEST_CONSTANTS.AUTH_TOKEN, HEALTH_CHECK_USER_AGENT); - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheckTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheckTest.java new file mode 100644 index 0000000000..6893797446 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheckTest.java @@ -0,0 +1,147 @@ +package io.stargate.sgv2.jsonapi.api.health; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.DefaultConsistencyLevel; +import com.datastax.oss.driver.api.core.cql.AsyncResultSet; +import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import io.smallrye.mutiny.TimeoutException; +import io.smallrye.mutiny.Uni; +import io.smallrye.mutiny.helpers.test.UniAssertSubscriber; +import io.stargate.sgv2.jsonapi.api.request.RequestContext; +import io.stargate.sgv2.jsonapi.api.request.UserAgent; +import io.stargate.sgv2.jsonapi.api.request.tenant.Tenant; +import io.stargate.sgv2.jsonapi.config.DatabaseType; +import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +public class DatabaseReadinessCheckTest { + + private static final Duration TIMEOUT = Duration.ofMillis(100); + private static final String ASTRA_TENANT_ID = "60b5dccb-e91d-4a60-987b-7588cd8aa1e3"; + + private CQLSessionCache sessionCache; + private CqlSession session; + private RequestContext requestContext; + private DatabaseReadinessCheck readinessCheck; + + @BeforeEach + public void setup() { + sessionCache = mock(CQLSessionCache.class); + session = mock(CqlSession.class); + requestContext = + new RequestContext( + Tenant.create(DatabaseType.ASTRA, ASTRA_TENANT_ID, "us-west-2"), + "astra-token", + new UserAgent("Datastax-SLA-Checker")); + readinessCheck = new DatabaseReadinessCheck(sessionCache, TIMEOUT); + } + + @Test + public void successfulCheckUsesRequestContextAndAsyncDistributedQuery() { + var resultSet = mock(AsyncResultSet.class); + var resultFuture = new CompletableFuture(); + when(sessionCache.getSession(requestContext)).thenReturn(Uni.createFrom().item(session)); + when(session.executeAsync(any(SimpleStatement.class))).thenReturn(resultFuture); + + var subscriber = + readinessCheck + .check(requestContext) + .subscribe() + .withSubscriber(UniAssertSubscriber.create()); + + subscriber.assertSubscribed().assertNotTerminated(); + resultFuture.complete(resultSet); + subscriber.awaitItem().assertItem(null).assertCompleted(); + + verify(sessionCache).getSession(same(requestContext)); + var statementCaptor = ArgumentCaptor.forClass(SimpleStatement.class); + verify(session).executeAsync(statementCaptor.capture()); + assertThat(statementCaptor.getValue().getQuery()) + .isEqualTo("SELECT * FROM datastax_sla.check LIMIT 1"); + assertThat(statementCaptor.getValue().getTimeout()).isEqualTo(TIMEOUT); + assertThat(statementCaptor.getValue().getConsistencyLevel()) + .isEqualTo(DefaultConsistencyLevel.LOCAL_QUORUM); + assertThat(statementCaptor.getValue().getExecutionProfileName()).isEqualTo("table-read"); + verify(session, never()).execute(any(SimpleStatement.class)); + verify(session, never()).isClosed(); + verify(sessionCache, never()).evictSession(any(RequestContext.class)); + } + + @Test + public void sessionAcquisitionFailureIsPropagated() { + when(sessionCache.getSession(requestContext)) + .thenReturn(Uni.createFrom().failure(new IllegalStateException("Cannot connect"))); + + readinessCheck + .check(requestContext) + .subscribe() + .withSubscriber(UniAssertSubscriber.create()) + .awaitFailure() + .assertFailedWith(IllegalStateException.class, "Cannot connect"); + + verify(session, never()).executeAsync(any(SimpleStatement.class)); + verify(sessionCache, never()).evictSession(any(RequestContext.class)); + } + + @Test + public void asynchronousQueryFailureIsPropagated() { + var resultFuture = new CompletableFuture(); + when(sessionCache.getSession(requestContext)).thenReturn(Uni.createFrom().item(session)); + when(session.executeAsync(any(SimpleStatement.class))).thenReturn(resultFuture); + + var subscriber = + readinessCheck + .check(requestContext) + .subscribe() + .withSubscriber(UniAssertSubscriber.create()); + resultFuture.completeExceptionally(new IllegalStateException("Query failed")); + + subscriber.awaitFailure().assertFailedWith(IllegalStateException.class, "Query failed"); + verify(sessionCache, never()).evictSession(any(RequestContext.class)); + } + + @Test + public void reactiveTimeoutBoundsSessionAcquisition() { + readinessCheck = new DatabaseReadinessCheck(sessionCache, Duration.ofMillis(20)); + when(sessionCache.getSession(requestContext)).thenReturn(Uni.createFrom().nothing()); + + readinessCheck + .check(requestContext) + .subscribe() + .withSubscriber(UniAssertSubscriber.create()) + .awaitFailure() + .assertFailedWith(TimeoutException.class); + + verify(session, never()).executeAsync(any(SimpleStatement.class)); + verify(sessionCache, never()).evictSession(any(RequestContext.class)); + } + + @Test + public void reactiveTimeoutBoundsAsynchronousQuery() { + readinessCheck = new DatabaseReadinessCheck(sessionCache, Duration.ofMillis(20)); + when(sessionCache.getSession(requestContext)).thenReturn(Uni.createFrom().item(session)); + when(session.executeAsync(any(SimpleStatement.class))).thenReturn(new CompletableFuture<>()); + + readinessCheck + .check(requestContext) + .subscribe() + .withSubscriber(UniAssertSubscriber.create()) + .awaitFailure() + .assertFailedWith(TimeoutException.class); + + verify(session).executeAsync(any(SimpleStatement.class)); + verify(sessionCache, never()).evictSession(any(RequestContext.class)); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResourceTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResourceTest.java new file mode 100644 index 0000000000..0fd4d9e8d3 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResourceTest.java @@ -0,0 +1,181 @@ +package io.stargate.sgv2.jsonapi.api.v1; + +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.AsyncResultSet; +import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import io.quarkus.security.UnauthorizedException; +import io.quarkus.test.InjectMock; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import io.smallrye.mutiny.Uni; +import io.stargate.sgv2.jsonapi.api.request.RequestContext; +import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; +import io.stargate.sgv2.jsonapi.exception.APISecurityException; +import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; +import io.stargate.sgv2.jsonapi.service.cqldriver.CqlSessionCacheSupplier; +import io.stargate.sgv2.jsonapi.testresource.NoGlobalResourcesTestProfile; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +@QuarkusTest +@TestProfile(DatabaseReadinessResourceTest.AstraProfile.class) +public class DatabaseReadinessResourceTest { + + private static final String TENANT_ID = "60b5dccb-e91d-4a60-987b-7588cd8aa1e3"; + private static final String REGION = "us-west-2"; + private static final String ASTRA_HOST = TENANT_ID + "-" + REGION + ".apps.astra.datastax.com"; + private static final String TOKEN = "astra-canary-token"; + private static final String SLA_USER_AGENT = "Datastax-SLA-Checker"; + + @InjectMock CqlSessionCacheSupplier sessionCacheSupplier; + + private CQLSessionCache sessionCache; + private CqlSession session; + + @BeforeEach + public void setup() { + sessionCache = mock(CQLSessionCache.class); + session = mock(CqlSession.class); + when(sessionCacheSupplier.get()).thenReturn(sessionCache); + } + + @Test + public void missingTokenIsRejectedBeforeDatabaseAccess() { + given() + .header("Host", ASTRA_HOST) + .header("User-Agent", SLA_USER_AGENT) + .when() + .get(DatabaseReadinessResource.BASE_PATH) + .then() + .statusCode(401); + + verifyNoInteractions(sessionCache); + } + + @Test + public void astraRequestUsesResolvedTenantTokenAndSlaUserAgent() { + var resultSet = mock(AsyncResultSet.class); + var capturedContext = new AtomicReference(); + when(sessionCache.getSession(any(RequestContext.class))) + .thenAnswer( + invocation -> { + RequestContext context = invocation.getArgument(0); + capturedContext.set( + new RequestContextSnapshot( + context.tenant().toString(), + context.tenant().region(), + context.authToken(), + context.userAgent().toString())); + return Uni.createFrom().item(session); + }); + when(session.executeAsync(any(SimpleStatement.class))) + .thenReturn(CompletableFuture.completedFuture(resultSet)); + + authenticatedRequest() + .when() + .get(DatabaseReadinessResource.BASE_PATH) + .then() + .statusCode(200) + .body("status", equalTo("UP")); + + assertThat(capturedContext.get()) + .isEqualTo(new RequestContextSnapshot(TENANT_ID, REGION, TOKEN, SLA_USER_AGENT)); + } + + @Test + public void databaseFailureReturnsServiceUnavailableWithoutDetails() { + when(sessionCache.getSession(any(RequestContext.class))) + .thenReturn(Uni.createFrom().failure(new IllegalStateException("sensitive failure"))); + + var response = + authenticatedRequest() + .when() + .get(DatabaseReadinessResource.BASE_PATH) + .then() + .statusCode(503) + .body("status", equalTo("DOWN")) + .extract() + .asString(); + + assertThat(response).doesNotContain("sensitive failure", TOKEN, TENANT_ID); + } + + @Test + public void invalidTokenFailureReturnsUnauthorizedWithoutDetails() { + when(sessionCache.getSession(any(RequestContext.class))) + .thenThrow(new UnauthorizedException("sensitive credential failure")); + + var response = + authenticatedRequest() + .when() + .get(DatabaseReadinessResource.BASE_PATH) + .then() + .statusCode(401) + .body("status", equalTo("DOWN")) + .extract() + .asString(); + + assertThat(response).doesNotContain("sensitive credential failure", TOKEN, TENANT_ID); + } + + @Test + public void databaseAuthenticationFailureReturnsUnauthorizedWithoutDetails() { + var authenticationFailure = + APISecurityException.Code.UNAUTHENTICATED_REQUEST.withPreformattedMessage( + "sensitive database authentication failure"); + when(sessionCache.getSession(any(RequestContext.class))) + .thenReturn(Uni.createFrom().failure(authenticationFailure)); + + var response = + authenticatedRequest() + .when() + .get(DatabaseReadinessResource.BASE_PATH) + .then() + .statusCode(401) + .body("status", equalTo("DOWN")) + .extract() + .asString(); + + assertThat(response) + .doesNotContain("sensitive database authentication failure", TOKEN, TENANT_ID); + } + + private io.restassured.specification.RequestSpecification authenticatedRequest() { + return given() + .header("Host", ASTRA_HOST) + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, TOKEN) + .header("User-Agent", SLA_USER_AGENT); + } + + private record RequestContextSnapshot( + String tenantId, String region, String authToken, String userAgent) {} + + public static class AstraProfile implements NoGlobalResourcesTestProfile { + + @Override + public Map getConfigOverrides() { + return Map.of( + "stargate.jsonapi.operations.database-config.type", + "ASTRA", + "stargate.multi-tenancy.enabled", + "true", + "stargate.multi-tenancy.tenant-resolver.type", + "subdomain", + "stargate.multi-tenancy.tenant-resolver.subdomain.max-chars", + "36", + "stargate.jsonapi.operations.sla-user-agent", + SLA_USER_AGENT); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java index 28ec1c2907..5afb578c28 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.net.ServerSocket; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -30,8 +31,7 @@ public class SessionEvictionIntegrationTest extends AbstractCollectionIntegratio private static final Logger LOGGER = LoggerFactory.getLogger(SessionEvictionIntegrationTest.class); - private static final String READINESS_PATH = "/stargate/health/ready"; - private static final String LIVENESS_PATH = "/stargate/health/live"; + private static final String READINESS_USER_AGENT = "DataAPI-Readiness-Test/1.0"; /** * Overridden to ensure we connect to the isolated container created for this test. @@ -52,6 +52,13 @@ protected int getCassandraCqlPort() { @Test public void testSessionEvictionOnAllNodesFailed() { + if (!executeCqlStatement( + "CREATE KEYSPACE IF NOT EXISTS datastax_sla " + + "WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}", + "CREATE TABLE IF NOT EXISTS datastax_sla.check (id text PRIMARY KEY)")) { + throw new AssertionError("Failed to provision the distributed readiness table"); + } + // 1. Insert and find initial data to ensure the database is healthy before the test insertDoc( """ @@ -91,7 +98,6 @@ public void testSessionEvictionOnAllNodesFailed() { "errors[0].errorCode", is(DatabaseException.Code.FAILED_TO_CONNECT_TO_DATABASE.name())); waitForReadinessStatus("DOWN", 60_000); - given().when().get(LIVENESS_PATH).then().statusCode(200).body("status", is("UP")); // 4. Restart the container to simulate recovery getDockerClient().startContainerCmd(getContainerId()).exec(); @@ -233,10 +239,7 @@ private boolean isApiReady() { } } - /** - * Polls the readiness endpoint until both the overall health and Cassandra connectivity check - * have the expected status. - */ + /** Polls the authenticated database readiness endpoint until it has the expected status. */ private void waitForReadinessStatus(String expectedStatus, long timeoutMillis) { var start = System.currentTimeMillis(); var expectedStatusCode = "UP".equals(expectedStatus) ? 200 : 503; @@ -244,15 +247,17 @@ private void waitForReadinessStatus(String expectedStatus, long timeoutMillis) { while (System.currentTimeMillis() - start < timeoutMillis) { try { - lastResponse = given().when().get(READINESS_PATH); + lastResponse = + given() + .headers(getHeaders()) + .header("User-Agent", READINESS_USER_AGENT) + .when() + .get(DatabaseReadinessResource.BASE_PATH); var jsonPath = lastResponse.jsonPath(); var overallStatus = jsonPath.getString("status"); - var cassandraStatus = - jsonPath.getString("checks.find { it.name == 'cassandra-connection' }.status"); if (lastResponse.statusCode() == expectedStatusCode - && expectedStatus.equals(overallStatus) - && expectedStatus.equals(cassandraStatus)) { + && expectedStatus.equals(overallStatus)) { return; } } catch (Exception e) { @@ -372,9 +377,10 @@ protected GenericContainer baseCassandraContainer(boolean reuse) { */ @Override public Map start() { - var props = super.start(); + var props = new HashMap<>(super.start()); + props.put("stargate.jsonapi.operations.sla-user-agent", READINESS_USER_AGENT); sessionEvictionCassandraContainer = super.getCassandraContainer(); - return props; + return Map.copyOf(props); } /** diff --git a/src/test/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilterTest.java b/src/test/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilterTest.java new file mode 100644 index 0000000000..5b9393c522 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilterTest.java @@ -0,0 +1,40 @@ +package io.stargate.sgv2.jsonapi.metrics; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import io.micrometer.core.instrument.MeterRegistry; +import io.stargate.sgv2.jsonapi.api.request.RequestContext; +import io.stargate.sgv2.jsonapi.api.v1.DatabaseReadinessResource; +import io.stargate.sgv2.jsonapi.api.v1.metrics.MetricsConfig; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerResponseContext; +import jakarta.ws.rs.core.UriInfo; +import java.net.URI; +import org.junit.jupiter.api.Test; + +public class TenantRequestMetricsFilterTest { + + @Test + public void databaseReadinessIsNotCountedAsTenantTraffic() { + var meterRegistry = mock(MeterRegistry.class); + var dataApiRequestContext = mock(RequestContext.class); + var metricsConfig = mock(MetricsConfig.class); + var tenantRequestConfig = mock(MetricsConfig.TenantRequestCounterConfig.class); + when(metricsConfig.tenantRequestCounter()).thenReturn(tenantRequestConfig); + when(tenantRequestConfig.enabled()).thenReturn(true); + + var requestContext = mock(ContainerRequestContext.class); + var uriInfo = mock(UriInfo.class); + when(requestContext.getUriInfo()).thenReturn(uriInfo); + when(uriInfo.getRequestUri()) + .thenReturn(URI.create("http://localhost" + DatabaseReadinessResource.BASE_PATH)); + + var filter = + new TenantRequestMetricsFilter(meterRegistry, dataApiRequestContext, metricsConfig); + filter.record(requestContext, mock(ContainerResponseContext.class)); + + verifyNoInteractions(meterRegistry, dataApiRequestContext); + } +} From 7423a237c0a6a0412df0bf098686878b5e4dfb9a Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Mon, 3 Aug 2026 20:50:06 -0700 Subject: [PATCH 07/12] fix: address review comments --- CONFIGURATION.md | 20 ++++-- .../api/health/DatabaseReadinessCheck.java | 1 + .../api/v1/DatabaseReadinessResource.java | 66 +++++++++++++---- .../metrics/TenantRequestMetricsFilter.java | 5 +- .../cqldriver/CqlSessionCacheSupplier.java | 12 +++- src/main/resources/application.yaml | 2 +- .../api/v1/DatabaseReadinessResourceTest.java | 70 +++++++++++++++++-- .../TenantRequestMetricsFilterTest.java | 11 +-- .../CqlSessionCacheSupplierTests.java | 34 +++++---- 9 files changed, 174 insertions(+), 47 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 70e0b43147..76b507e757 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -71,18 +71,26 @@ provision a `datastax_sla.check` table that the canary principal can read. Its r must be appropriate for the deployment (greater than one in a multi-node local data center) so `LOCAL_QUORUM` requires responses from multiple replicas. Astra callers must use the canary database hostname so the tenant and region are resolved from `Host`; Cassandra ignores the tenant portion of -`Host`. The caller must also send the exact User-Agent configured by -`stargate.jsonapi.operations.sla-user-agent`, allowing a dedicated canary session to use the shorter -SLA session TTL instead of being treated like normal client traffic. +`Host`. The caller must also send the full User-Agent configured by +`stargate.jsonapi.operations.sla-user-agent`. The comparison is case-insensitive. Requests with a +missing or different User-Agent are rejected before accessing the session cache, and the endpoint +fails closed when the SLA User-Agent is not configured. This ensures the canary session uses the +shorter SLA session TTL instead of being treated like normal client traffic. Do not reuse the canary +credentials for normal traffic, because using the same cached session with a non-SLA User-Agent can +extend its lifetime. The check is fully asynchronous and has a five-second timeout. It returns HTTP 200 with -`{"status":"UP"}` after a successful read, HTTP 503 with `{"status":"DOWN"}` after a database -failure or timeout, and HTTP 401 when the `Token` header is missing or authentication fails. +`{"status":"UP"}` after a successful read and HTTP 503 with `{"status":"DOWN"}` after a database +failure, timeout, or missing SLA User-Agent configuration. It returns the standard Data API error +response with HTTP 401 when the `Token` header is missing or authentication fails, and HTTP 403 when +the request User-Agent does not match the configured SLA User-Agent. Probe integrations must use the +HTTP status as the readiness contract rather than parsing the response body's `status` field alone. Kubernetes or an SLA checker must call each pod directly for this endpoint to control per-pod readiness. An external request sent through a load balancer does not establish which pod is ready. Restrict the endpoint to trusted probe traffic with deployment controls such as a NetworkPolicy, -mTLS, or an ingress ACL and rate limit. Kubernetes `httpGet` headers cannot reference a Secret, so +mTLS, or an ingress ACL and rate limit. The User-Agent check is an operational guard, not an +authentication boundary. Kubernetes `httpGet` headers cannot reference a Secret, so delivery of the canary token is intentionally outside the Data API configuration. Prefer an external checker or a Secret-mounted file read by an `exec` probe; do not put the token literally in the probe command or shell trace. The unauthenticated Quarkus health endpoints under the diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java b/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java index bbec7687d1..6f4c3e0d50 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java @@ -65,6 +65,7 @@ public Uni check(RequestContext requestContext) { .executeRead(statement) .replaceWithVoid(); }) + // The statement timeout bounds driver I/O; this also bounds asynchronous session lookup. .ifNoItem() .after(timeout) .fail(); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java index 38bafaa95f..7e39b952c2 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java @@ -3,6 +3,8 @@ import io.quarkus.security.UnauthorizedException; import io.smallrye.mutiny.Uni; import io.stargate.sgv2.jsonapi.api.health.DatabaseReadinessCheck; +import io.stargate.sgv2.jsonapi.api.model.command.CommandResult; +import io.stargate.sgv2.jsonapi.api.model.command.tracing.RequestTracing; import io.stargate.sgv2.jsonapi.api.request.RequestContext; import io.stargate.sgv2.jsonapi.config.constants.OpenApiConstants; import io.stargate.sgv2.jsonapi.exception.APISecurityException; @@ -13,6 +15,8 @@ import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import java.util.Collections; +import java.util.IdentityHashMap; import org.eclipse.microprofile.openapi.annotations.Operation; import org.eclipse.microprofile.openapi.annotations.media.Content; import org.eclipse.microprofile.openapi.annotations.media.Schema; @@ -25,9 +29,10 @@ * Authenticated database readiness endpoint registered through Quarkus JAX-RS resource discovery. * *

{@code GET /v1/health/ready} runs the same request-scoped probe for Astra and Cassandra. A - * successful probe returns HTTP 200; invalid credentials return HTTP 401; and a database failure or - * timeout returns HTTP 503. The existing {@code /v1/*} security policy rejects requests without a - * token before this resource is called. + * request must use the configured SLA User-Agent. A successful probe returns HTTP 200; invalid + * credentials return HTTP 401; a missing or different SLA User-Agent returns HTTP 403; and a + * database failure, timeout, or missing SLA configuration returns HTTP 503. The existing {@code + * /v1/*} security policy rejects requests without a token before this resource is called. */ @Path(DatabaseReadinessResource.BASE_PATH) @Produces(MediaType.APPLICATION_JSON) @@ -41,12 +46,14 @@ public class DatabaseReadinessResource { private final DatabaseReadinessCheck readinessCheck; private final RequestContext requestContext; + private final CqlSessionCacheSupplier sessionCacheSupplier; @Inject public DatabaseReadinessResource( CqlSessionCacheSupplier sessionCacheSupplier, RequestContext requestContext) { this.readinessCheck = new DatabaseReadinessCheck(sessionCacheSupplier); this.requestContext = requestContext; + this.sessionCacheSupplier = sessionCacheSupplier; } @GET @@ -62,33 +69,46 @@ public DatabaseReadinessResource( @Content( mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ReadinessResponse.class))), - @APIResponse(responseCode = "401", description = "The token is missing or invalid."), + @APIResponse( + responseCode = "401", + description = "The token is missing or invalid.", + content = + @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema(implementation = CommandResult.class))), + @APIResponse( + responseCode = "403", + description = "The request does not use the configured SLA User-Agent."), @APIResponse( responseCode = "503", - description = "The database read failed or timed out.", + description = "The SLA User-Agent is not configured, or the database check failed.", content = @Content( mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ReadinessResponse.class))) }) - public Uni> ready() { + public Uni> ready() { + var configuredSlaUserAgent = sessionCacheSupplier.slaUserAgent(); + if (configuredSlaUserAgent.isEmpty()) { + return Uni.createFrom().item(response(Response.Status.SERVICE_UNAVAILABLE, DOWN)); + } + if (!configuredSlaUserAgent.get().equals(requestContext.userAgent())) { + return Uni.createFrom().item(response(Response.Status.FORBIDDEN)); + } + return readinessCheck .check(requestContext) - .map(ignored -> RestResponse.ok(UP)) + .map(ignored -> response(Response.Status.OK, UP)) .onFailure(DatabaseReadinessResource::isUnauthorized) - .recoverWithItem( - failure -> - RestResponse.ResponseBuilder.create(Response.Status.UNAUTHORIZED, DOWN).build()) + .recoverWithItem(failure -> unauthorizedResponse()) .onFailure() - .recoverWithItem( - failure -> - RestResponse.ResponseBuilder.create(Response.Status.SERVICE_UNAVAILABLE, DOWN) - .build()); + .recoverWithItem(failure -> response(Response.Status.SERVICE_UNAVAILABLE, DOWN)); } private static boolean isUnauthorized(Throwable failure) { var current = failure; - while (current != null) { + var seen = Collections.newSetFromMap(new IdentityHashMap()); + while (current != null && seen.add(current)) { if (current instanceof UnauthorizedException || current instanceof APISecurityException apiException && apiException.httpStatus == Response.Status.UNAUTHORIZED.getStatusCode()) { @@ -99,5 +119,21 @@ private static boolean isUnauthorized(Throwable failure) { return false; } + private static RestResponse unauthorizedResponse() { + var commandResult = + CommandResult.statusOnlyBuilder(RequestTracing.NO_OP) + .addThrowable(APISecurityException.Code.UNAUTHENTICATED_REQUEST.get()) + .build(); + return response(Response.Status.UNAUTHORIZED, commandResult); + } + + private static RestResponse response(Response.Status status) { + return RestResponse.ResponseBuilder.create(status).build(); + } + + private static RestResponse response(Response.Status status, Object entity) { + return RestResponse.ResponseBuilder.create(status, entity).build(); + } + public record ReadinessResponse(String status) {} } diff --git a/src/main/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilter.java b/src/main/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilter.java index c633208225..b277403ff3 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilter.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilter.java @@ -100,8 +100,9 @@ public void record( } private static boolean isDatabaseReadinessRequest(ContainerRequestContext requestContext) { - return DatabaseReadinessResource.BASE_PATH.equals( - requestContext.getUriInfo().getRequestUri().getPath()); + var requestPath = requestContext.getUriInfo().getRequestUri().getPath(); + return DatabaseReadinessResource.BASE_PATH.equals(requestPath) + || (DatabaseReadinessResource.BASE_PATH + "/").equals(requestPath); } private String getUserAgentValue(ContainerRequestContext requestContext) { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionCacheSupplier.java b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionCacheSupplier.java index 766c36899f..7e569d60e4 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionCacheSupplier.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionCacheSupplier.java @@ -10,6 +10,7 @@ import java.time.Duration; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.function.Supplier; import org.eclipse.microprofile.config.inject.ConfigProperty; @@ -23,6 +24,7 @@ public class CqlSessionCacheSupplier implements Supplier { private final CQLSessionCache singleton; + private final Optional slaUserAgent; @Inject public CqlSessionCacheSupplier( @@ -53,11 +55,14 @@ public CqlSessionCacheSupplier( dbConfig.cassandraPort(), () -> schemaObjectCacheSupplier.get().getSchemaChangeListener()); + slaUserAgent = + operationsConfig.slaUserAgent().filter(value -> !value.isBlank()).map(UserAgent::new); + singleton = new CQLSessionCache( dbConfig.sessionCacheMaxSize(), Duration.ofSeconds(dbConfig.sessionCacheTtlSeconds()), - operationsConfig.slaUserAgent().map(UserAgent::new).orElse(null), + slaUserAgent.orElse(null), Duration.ofSeconds(dbConfig.slaSessionCacheTtlSeconds()), credentialsFactory, sessionFactory, @@ -70,4 +75,9 @@ public CqlSessionCacheSupplier( public CQLSessionCache get() { return singleton; } + + /** Gets the configured User-Agent that selects the shorter SLA session-cache TTL. */ + public Optional slaUserAgent() { + return slaUserAgent; + } } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 1103fef1b3..a978343bbc 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -159,7 +159,7 @@ quarkus: http-server: # ignore all non-application uris, as well as the custom set suppress-non-application-uris: true - ignore-patterns: /,/metrics,/swagger-ui.*,.*\.html,/v1/health/ready + ignore-patterns: /,/metrics,/swagger-ui.*,.*\.html,/v1/health/ready/? # due to the https://github.com/quarkusio/quarkus/issues/24938 # we need to define uri templating on our own for now diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResourceTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResourceTest.java index 0fd4d9e8d3..c3db7999f8 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResourceTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResourceTest.java @@ -17,16 +17,20 @@ import io.quarkus.test.junit.TestProfile; import io.smallrye.mutiny.Uni; import io.stargate.sgv2.jsonapi.api.request.RequestContext; +import io.stargate.sgv2.jsonapi.api.request.UserAgent; import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; import io.stargate.sgv2.jsonapi.exception.APISecurityException; import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; import io.stargate.sgv2.jsonapi.service.cqldriver.CqlSessionCacheSupplier; import io.stargate.sgv2.jsonapi.testresource.NoGlobalResourcesTestProfile; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; @QuarkusTest @TestProfile(DatabaseReadinessResourceTest.AstraProfile.class) @@ -48,6 +52,8 @@ public void setup() { sessionCache = mock(CQLSessionCache.class); session = mock(CqlSession.class); when(sessionCacheSupplier.get()).thenReturn(sessionCache); + when(sessionCacheSupplier.slaUserAgent()) + .thenReturn(Optional.of(new UserAgent(SLA_USER_AGENT))); } @Test @@ -58,13 +64,15 @@ public void missingTokenIsRejectedBeforeDatabaseAccess() { .when() .get(DatabaseReadinessResource.BASE_PATH) .then() - .statusCode(401); + .statusCode(401) + .body("errors[0].errorCode", equalTo("MISSING_AUTHENTICATION_TOKEN")); verifyNoInteractions(sessionCache); } - @Test - public void astraRequestUsesResolvedTenantTokenAndSlaUserAgent() { + @ParameterizedTest + @ValueSource(strings = {"", "/"}) + public void astraRequestUsesResolvedTenantTokenAndSlaUserAgent(String pathSuffix) { var resultSet = mock(AsyncResultSet.class); var capturedContext = new AtomicReference(); when(sessionCache.getSession(any(RequestContext.class))) @@ -84,7 +92,7 @@ public void astraRequestUsesResolvedTenantTokenAndSlaUserAgent() { authenticatedRequest() .when() - .get(DatabaseReadinessResource.BASE_PATH) + .get(DatabaseReadinessResource.BASE_PATH + pathSuffix) .then() .statusCode(200) .body("status", equalTo("UP")); @@ -93,6 +101,34 @@ public void astraRequestUsesResolvedTenantTokenAndSlaUserAgent() { .isEqualTo(new RequestContextSnapshot(TENANT_ID, REGION, TOKEN, SLA_USER_AGENT)); } + @Test + public void wrongSlaUserAgentIsRejectedBeforeDatabaseAccess() { + given() + .header("Host", ASTRA_HOST) + .header(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, TOKEN) + .header("User-Agent", "ordinary-client") + .when() + .get(DatabaseReadinessResource.BASE_PATH) + .then() + .statusCode(403); + + verifyNoInteractions(sessionCache); + } + + @Test + public void missingSlaUserAgentConfigurationReturnsServiceUnavailable() { + when(sessionCacheSupplier.slaUserAgent()).thenReturn(Optional.empty()); + + authenticatedRequest() + .when() + .get(DatabaseReadinessResource.BASE_PATH) + .then() + .statusCode(503) + .body("status", equalTo("DOWN")); + + verifyNoInteractions(sessionCache); + } + @Test public void databaseFailureReturnsServiceUnavailableWithoutDetails() { when(sessionCache.getSession(any(RequestContext.class))) @@ -122,7 +158,7 @@ public void invalidTokenFailureReturnsUnauthorizedWithoutDetails() { .get(DatabaseReadinessResource.BASE_PATH) .then() .statusCode(401) - .body("status", equalTo("DOWN")) + .body("errors[0].errorCode", equalTo("UNAUTHENTICATED_REQUEST")) .extract() .asString(); @@ -143,7 +179,7 @@ public void databaseAuthenticationFailureReturnsUnauthorizedWithoutDetails() { .get(DatabaseReadinessResource.BASE_PATH) .then() .statusCode(401) - .body("status", equalTo("DOWN")) + .body("errors[0].errorCode", equalTo("UNAUTHENTICATED_REQUEST")) .extract() .asString(); @@ -151,6 +187,28 @@ public void databaseAuthenticationFailureReturnsUnauthorizedWithoutDetails() { .doesNotContain("sensitive database authentication failure", TOKEN, TENANT_ID); } + @Test + public void cyclicFailureCauseReturnsServiceUnavailable() { + var firstFailure = new IllegalStateException("first sensitive failure"); + var secondFailure = new IllegalArgumentException("second sensitive failure"); + firstFailure.initCause(secondFailure); + secondFailure.initCause(firstFailure); + when(sessionCache.getSession(any(RequestContext.class))) + .thenReturn(Uni.createFrom().failure(firstFailure)); + + var response = + authenticatedRequest() + .when() + .get(DatabaseReadinessResource.BASE_PATH) + .then() + .statusCode(503) + .body("status", equalTo("DOWN")) + .extract() + .asString(); + + assertThat(response).doesNotContain("first sensitive failure", "second sensitive failure"); + } + private io.restassured.specification.RequestSpecification authenticatedRequest() { return given() .header("Host", ASTRA_HOST) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilterTest.java b/src/test/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilterTest.java index 5b9393c522..648e2bcaf2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilterTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/metrics/TenantRequestMetricsFilterTest.java @@ -12,12 +12,14 @@ import jakarta.ws.rs.container.ContainerResponseContext; import jakarta.ws.rs.core.UriInfo; import java.net.URI; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; public class TenantRequestMetricsFilterTest { - @Test - public void databaseReadinessIsNotCountedAsTenantTraffic() { + @ParameterizedTest + @ValueSource(strings = {"", "/"}) + public void databaseReadinessIsNotCountedAsTenantTraffic(String pathSuffix) { var meterRegistry = mock(MeterRegistry.class); var dataApiRequestContext = mock(RequestContext.class); var metricsConfig = mock(MetricsConfig.class); @@ -29,7 +31,8 @@ public void databaseReadinessIsNotCountedAsTenantTraffic() { var uriInfo = mock(UriInfo.class); when(requestContext.getUriInfo()).thenReturn(uriInfo); when(uriInfo.getRequestUri()) - .thenReturn(URI.create("http://localhost" + DatabaseReadinessResource.BASE_PATH)); + .thenReturn( + URI.create("http://localhost" + DatabaseReadinessResource.BASE_PATH + pathSuffix)); var filter = new TenantRequestMetricsFilter(meterRegistry, dataApiRequestContext, metricsConfig); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionCacheSupplierTests.java b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionCacheSupplierTests.java index de0cfca852..976344d441 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionCacheSupplierTests.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlSessionCacheSupplierTests.java @@ -7,6 +7,7 @@ import com.datastax.oss.driver.api.core.metadata.schema.SchemaChangeListener; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import io.stargate.sgv2.jsonapi.TestConstants; +import io.stargate.sgv2.jsonapi.api.request.UserAgent; import io.stargate.sgv2.jsonapi.config.DatabaseType; import io.stargate.sgv2.jsonapi.config.OperationsConfig; import io.stargate.sgv2.jsonapi.service.schema.SchemaObjectCache; @@ -24,6 +25,24 @@ public class CqlSessionCacheSupplierTests { public void testSingleton() { // Not a lot to test, just checking it always returns the same instance. + var factory = newFactory(Optional.of(TEST_CONSTANTS.SLA_USER_AGENT_NAME)); + + var sessionCache1 = factory.get(); + var sessionCache2 = factory.get(); + + assertThat(sessionCache1) + .as("Session cache should be the same instance") + .isSameAs(sessionCache2); + assertThat(factory.slaUserAgent()).contains(new UserAgent(TEST_CONSTANTS.SLA_USER_AGENT_NAME)); + } + + @Test + public void blankSlaUserAgentIsTreatedAsUnconfigured() { + assertThat(newFactory(Optional.of(" ")).slaUserAgent()).isEmpty(); + } + + private CqlSessionCacheSupplier newFactory(Optional slaUserAgent) { + var dbConfig = mock(OperationsConfig.DatabaseConfig.class); when(dbConfig.type()).thenReturn(DatabaseType.ASTRA); when(dbConfig.localDatacenter()).thenReturn("datacenter1"); @@ -35,8 +54,7 @@ public void testSingleton() { var operationsConfig = mock(OperationsConfig.class); when(operationsConfig.databaseConfig()).thenReturn(dbConfig); - when(operationsConfig.slaUserAgent()) - .thenReturn(Optional.of(TEST_CONSTANTS.SLA_USER_AGENT_NAME)); + when(operationsConfig.slaUserAgent()).thenReturn(slaUserAgent); var mockSchemaObjectCacheSupplier = mock(SchemaObjectCacheSupplier.class); var mockSchemaObjectCache = mock(SchemaObjectCache.class); @@ -44,15 +62,7 @@ public void testSingleton() { when(mockSchemaObjectCache.getSchemaChangeListener()) .thenReturn(mock(SchemaChangeListener.class)); - var factory = - new CqlSessionCacheSupplier( - "testApp", operationsConfig, new SimpleMeterRegistry(), mockSchemaObjectCacheSupplier); - - var sessionCache1 = factory.get(); - var sessionCache2 = factory.get(); - - assertThat(sessionCache1) - .as("Session cache should be the same instance") - .isSameAs(sessionCache2); + return new CqlSessionCacheSupplier( + "testApp", operationsConfig, new SimpleMeterRegistry(), mockSchemaObjectCacheSupplier); } } From 987f7077eca2afadd806e4abbcbdb7ba69776210 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Mon, 10 Aug 2026 15:24:23 -0700 Subject: [PATCH 08/12] refactor: determine database readiness from driver session metadata Replace the datastax_sla.check canary query with a check of the driver session metadata: the pod is ready when the session obtained through the normal session cache reports at least one UP node. No query is issued and no canary table needs to be provisioned. Add stargate.jsonapi.operations.database-config.require-session-node-metadata (default false): when enabled, CqlSessionFactory rejects a newly built session whose metadata contains no nodes, closing it and failing creation with the standard FAILED_TO_CONNECT_TO_DATABASE error so an unusable session is never cached. --- CONFIGURATION.md | 55 +++++++---- .../api/health/DatabaseReadinessCheck.java | 39 +++++--- .../api/v1/DatabaseReadinessResource.java | 17 ++-- .../health/DatabaseReadinessCheckTest.java | 99 +++++++++---------- .../api/v1/DatabaseReadinessResourceTest.java | 38 +++++-- .../v1/SessionEvictionIntegrationTest.java | 7 -- .../CqlDriverConfigLoadEnvTests.java | 2 +- 7 files changed, 149 insertions(+), 108 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 76b507e757..0ff18f8181 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -61,37 +61,52 @@ Cassandra deployments. It uses the request's tenant, `Token` header, and `User-A session through the normal session cache. The Data API does not store separate readiness credentials. -The endpoint executes `SELECT * FROM datastax_sla.check LIMIT 1` at `LOCAL_QUORUM`, using the -`table-read` driver profile for the remaining read settings. An `UP` response therefore confirms -that the coordinator can complete a read from a replicated table at local quorum. It does not -validate every tenant's credentials, write availability, or cross-region availability. - -The deployment must provide a dedicated canary tenant and credentials for this request and must -provision a `datastax_sla.check` table that the canary principal can read. Its replication factor -must be appropriate for the deployment (greater than one in a multi-node local data center) so -`LOCAL_QUORUM` requires responses from multiple replicas. Astra callers must use the canary database -hostname so the tenant and region are resolved from `Host`; Cassandra ignores the tenant portion of -`Host`. The caller must also send the full User-Agent configured by +The check is based on the driver's session metadata, which the driver populates from +`system.local` and `system.peers` when the session connects and keeps current through node state +events. The pod is ready when the session metadata reports at least one node in the `UP` state. No +query is issued against the database: acquiring the session is itself part of the check, because a +pod that cannot connect to the database fails session creation and reports `DOWN`. An `UP` response +therefore confirms that this pod holds a usable session with a live connection to the database. It +does not validate every tenant's credentials, quorum availability, write availability, or +cross-region availability. + +Session creation itself carries a second, independent guard: `CqlSessionFactory` rejects any newly +built session whose driver metadata is missing the `system` keyspace, so a session that connected +but could not read the schema is never cached or handed to a request. That guard is always on and +applies to all session creation, not only readiness requests. + +The two checks cover different moments, and the interaction matters for probe configuration. The +factory guard runs once, when a session is created; the readiness check runs on every probe against +whatever session is cached. Because a failed session creation surfaces as a failed session +acquisition, a probe token that authenticates but cannot read the schema tables makes this endpoint +report `DOWN` rather than an authorization error. Since readiness drives pod rotation, a probe token +whose schema-read permission is revoked will take every pod out of service. Grant the probe token +schema read access and alert on a fleet-wide `DOWN` transition, which indicates a credential problem +rather than a database outage. + +The caller must send the full User-Agent configured by `stargate.jsonapi.operations.sla-user-agent`. The comparison is case-insensitive. Requests with a missing or different User-Agent are rejected before accessing the session cache, and the endpoint -fails closed when the SLA User-Agent is not configured. This ensures the canary session uses the -shorter SLA session TTL instead of being treated like normal client traffic. Do not reuse the canary +fails closed when the SLA User-Agent is not configured. This ensures the probe session uses the +shorter SLA session TTL instead of being treated like normal client traffic. Do not reuse the probe credentials for normal traffic, because using the same cached session with a non-SLA User-Agent can -extend its lifetime. +extend its lifetime. Astra callers must use the probe database hostname so the tenant and region are +resolved from `Host`; Cassandra ignores the tenant portion of `Host`. The check is fully asynchronous and has a five-second timeout. It returns HTTP 200 with -`{"status":"UP"}` after a successful read and HTTP 503 with `{"status":"DOWN"}` after a database -failure, timeout, or missing SLA User-Agent configuration. It returns the standard Data API error -response with HTTP 401 when the `Token` header is missing or authentication fails, and HTTP 403 when -the request User-Agent does not match the configured SLA User-Agent. Probe integrations must use the -HTTP status as the readiness contract rather than parsing the response body's `status` field alone. +`{"status":"UP"}` when the session metadata reports an `UP` node and HTTP 503 with +`{"status":"DOWN"}` after a session failure, timeout, or missing SLA User-Agent configuration. It +returns the standard Data API error response with HTTP 401 when the `Token` header is missing or +authentication fails, and HTTP 403 when the request User-Agent does not match the configured SLA +User-Agent. Probe integrations must use the HTTP status as the readiness contract rather than +parsing the response body's `status` field alone. Kubernetes or an SLA checker must call each pod directly for this endpoint to control per-pod readiness. An external request sent through a load balancer does not establish which pod is ready. Restrict the endpoint to trusted probe traffic with deployment controls such as a NetworkPolicy, mTLS, or an ingress ACL and rate limit. The User-Agent check is an operational guard, not an authentication boundary. Kubernetes `httpGet` headers cannot reference a Secret, so -delivery of the canary token is intentionally outside the Data API configuration. Prefer an +delivery of the probe token is intentionally outside the Data API configuration. Prefer an external checker or a Secret-mounted file read by an `exec` probe; do not put the token literally in the probe command or shell trace. The unauthenticated Quarkus health endpoints under the non-application path do not include this database check. diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java b/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java index 6f4c3e0d50..a83baaf7c2 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java @@ -1,12 +1,10 @@ package io.stargate.sgv2.jsonapi.api.health; -import com.datastax.oss.driver.api.core.DefaultConsistencyLevel; -import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import com.datastax.oss.driver.api.core.metadata.NodeState; import com.google.common.annotations.VisibleForTesting; import io.smallrye.mutiny.Uni; import io.stargate.sgv2.jsonapi.api.request.RequestContext; import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; -import io.stargate.sgv2.jsonapi.service.cqldriver.executor.CommandQueryExecutor; import java.time.Duration; import java.util.Objects; import java.util.function.Supplier; @@ -14,17 +12,22 @@ /** * Runs the database probe exposed at {@code GET /v1/health/ready}. * + *

The probe obtains a session through the normal session cache and inspects the driver session + * metadata, which the driver populates from {@code system.local} and {@code system.peers} and keeps + * current through node state events. The pod is ready when at least one node is {@link + * NodeState#UP}. Acquiring the session is itself part of the check: a pod that cannot connect to + * the database fails session creation and reports {@code DOWN}, and no query is issued against the + * database. + * *

This class is constructed by the JAX-RS resource and is not a CDI bean or a MicroProfile * health check. The caller's request context supplies the tenant, token, and User-Agent for both * Astra and Cassandra connections. */ public final class DatabaseReadinessCheck { - private static final String READINESS_QUERY = "SELECT * FROM datastax_sla.check LIMIT 1"; private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(5); private final Supplier sessionCacheSupplier; - private final SimpleStatement statement; private final Duration timeout; public DatabaseReadinessCheck(Supplier sessionCacheSupplier) { @@ -40,16 +43,11 @@ private DatabaseReadinessCheck(Supplier sessionCacheSupplier, D this.sessionCacheSupplier = Objects.requireNonNull(sessionCacheSupplier, "sessionCacheSupplier must not be null"); this.timeout = Objects.requireNonNull(timeout, "timeout must not be null"); - this.statement = - SimpleStatement.builder(READINESS_QUERY) - .setConsistencyLevel(DefaultConsistencyLevel.LOCAL_QUORUM) - .setTimeout(timeout) - .build(); } /** - * Executes a replicated table read at {@code LOCAL_QUORUM}, using the {@code table-read} driver - * profile for the remaining read settings. + * Obtains a session from the cache and fails unless the session metadata has at least one node in + * the {@link NodeState#UP} state. */ public Uni check(RequestContext requestContext) { Objects.requireNonNull(requestContext, "requestContext must not be null"); @@ -60,12 +58,21 @@ public Uni check(RequestContext requestContext) { var sessionCache = Objects.requireNonNull( sessionCacheSupplier.get(), "sessionCacheSupplier returned null"); - return new CommandQueryExecutor( - sessionCache, requestContext, CommandQueryExecutor.QueryTarget.TABLE) - .executeRead(statement) + return sessionCache + .getSession(requestContext) + .invoke( + session -> { + var anyNodeUp = + session.getMetadata().getNodes().values().stream() + .anyMatch(node -> node.getState() == NodeState.UP); + if (!anyNodeUp) { + throw new IllegalStateException( + "Session metadata has no node in the UP state"); + } + }) .replaceWithVoid(); }) - // The statement timeout bounds driver I/O; this also bounds asynchronous session lookup. + // Bounds asynchronous session acquisition, the metadata inspection is in-memory only. .ifNoItem() .after(timeout) .fail(); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java index 7e39b952c2..078402ddf1 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java @@ -28,11 +28,13 @@ /** * Authenticated database readiness endpoint registered through Quarkus JAX-RS resource discovery. * - *

{@code GET /v1/health/ready} runs the same request-scoped probe for Astra and Cassandra. A - * request must use the configured SLA User-Agent. A successful probe returns HTTP 200; invalid - * credentials return HTTP 401; a missing or different SLA User-Agent returns HTTP 403; and a - * database failure, timeout, or missing SLA configuration returns HTTP 503. The existing {@code - * /v1/*} security policy rejects requests without a token before this resource is called. + *

{@code GET /v1/health/ready} runs the same request-scoped probe for Astra and Cassandra: it + * obtains a session through the normal session cache and checks the driver session metadata for an + * UP node, see {@link DatabaseReadinessCheck}. A request must use the configured SLA User-Agent. A + * successful probe returns HTTP 200; invalid credentials return HTTP 401; a missing or different + * SLA User-Agent returns HTTP 403; and a session failure, timeout, or missing SLA configuration + * returns HTTP 503. The existing {@code /v1/*} security policy rejects requests without a token + * before this resource is called. */ @Path(DatabaseReadinessResource.BASE_PATH) @Produces(MediaType.APPLICATION_JSON) @@ -60,11 +62,12 @@ public DatabaseReadinessResource( @Operation( summary = "Check database readiness", description = - "Uses the authenticated request tenant and token to perform a LOCAL_QUORUM read.") + "Uses the authenticated request tenant and token to obtain a session and checks the" + + " driver session metadata for an UP node.") @APIResponses({ @APIResponse( responseCode = "200", - description = "The database completed the readiness read.", + description = "The session metadata reports at least one UP node.", content = @Content( mediaType = MediaType.APPLICATION_JSON, diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheckTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheckTest.java index 6893797446..8fd8c4d619 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheckTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheckTest.java @@ -1,6 +1,5 @@ package io.stargate.sgv2.jsonapi.api.health; -import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -9,9 +8,10 @@ import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.CqlSession; -import com.datastax.oss.driver.api.core.DefaultConsistencyLevel; -import com.datastax.oss.driver.api.core.cql.AsyncResultSet; import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import com.datastax.oss.driver.api.core.metadata.Metadata; +import com.datastax.oss.driver.api.core.metadata.Node; +import com.datastax.oss.driver.api.core.metadata.NodeState; import io.smallrye.mutiny.TimeoutException; import io.smallrye.mutiny.Uni; import io.smallrye.mutiny.helpers.test.UniAssertSubscriber; @@ -21,10 +21,11 @@ import io.stargate.sgv2.jsonapi.config.DatabaseType; import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; import java.time.Duration; -import java.util.concurrent.CompletableFuture; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; public class DatabaseReadinessCheckTest { @@ -48,91 +49,87 @@ public void setup() { readinessCheck = new DatabaseReadinessCheck(sessionCache, TIMEOUT); } + private void stubSessionMetadata(NodeState... nodeStates) { + var metadata = mock(Metadata.class); + var nodes = new HashMap(); + for (NodeState nodeState : nodeStates) { + var node = mock(Node.class); + when(node.getState()).thenReturn(nodeState); + nodes.put(UUID.randomUUID(), node); + } + when(metadata.getNodes()).thenReturn(Map.copyOf(nodes)); + when(session.getMetadata()).thenReturn(metadata); + } + @Test - public void successfulCheckUsesRequestContextAndAsyncDistributedQuery() { - var resultSet = mock(AsyncResultSet.class); - var resultFuture = new CompletableFuture(); + public void upNodeInSessionMetadataCompletesWithoutQuerying() { + stubSessionMetadata(NodeState.DOWN, NodeState.UP); when(sessionCache.getSession(requestContext)).thenReturn(Uni.createFrom().item(session)); - when(session.executeAsync(any(SimpleStatement.class))).thenReturn(resultFuture); - var subscriber = - readinessCheck - .check(requestContext) - .subscribe() - .withSubscriber(UniAssertSubscriber.create()); - - subscriber.assertSubscribed().assertNotTerminated(); - resultFuture.complete(resultSet); - subscriber.awaitItem().assertItem(null).assertCompleted(); + readinessCheck + .check(requestContext) + .subscribe() + .withSubscriber(UniAssertSubscriber.create()) + .awaitItem() + .assertItem(null) + .assertCompleted(); verify(sessionCache).getSession(same(requestContext)); - var statementCaptor = ArgumentCaptor.forClass(SimpleStatement.class); - verify(session).executeAsync(statementCaptor.capture()); - assertThat(statementCaptor.getValue().getQuery()) - .isEqualTo("SELECT * FROM datastax_sla.check LIMIT 1"); - assertThat(statementCaptor.getValue().getTimeout()).isEqualTo(TIMEOUT); - assertThat(statementCaptor.getValue().getConsistencyLevel()) - .isEqualTo(DefaultConsistencyLevel.LOCAL_QUORUM); - assertThat(statementCaptor.getValue().getExecutionProfileName()).isEqualTo("table-read"); + verify(session, never()).executeAsync(any(SimpleStatement.class)); verify(session, never()).execute(any(SimpleStatement.class)); - verify(session, never()).isClosed(); verify(sessionCache, never()).evictSession(any(RequestContext.class)); } @Test - public void sessionAcquisitionFailureIsPropagated() { - when(sessionCache.getSession(requestContext)) - .thenReturn(Uni.createFrom().failure(new IllegalStateException("Cannot connect"))); + public void noUpNodeInSessionMetadataFails() { + stubSessionMetadata(NodeState.DOWN, NodeState.UNKNOWN); + when(sessionCache.getSession(requestContext)).thenReturn(Uni.createFrom().item(session)); readinessCheck .check(requestContext) .subscribe() .withSubscriber(UniAssertSubscriber.create()) .awaitFailure() - .assertFailedWith(IllegalStateException.class, "Cannot connect"); + .assertFailedWith(IllegalStateException.class, "no node in the UP state"); - verify(session, never()).executeAsync(any(SimpleStatement.class)); verify(sessionCache, never()).evictSession(any(RequestContext.class)); } @Test - public void asynchronousQueryFailureIsPropagated() { - var resultFuture = new CompletableFuture(); + public void emptySessionMetadataFails() { + stubSessionMetadata(); when(sessionCache.getSession(requestContext)).thenReturn(Uni.createFrom().item(session)); - when(session.executeAsync(any(SimpleStatement.class))).thenReturn(resultFuture); - var subscriber = - readinessCheck - .check(requestContext) - .subscribe() - .withSubscriber(UniAssertSubscriber.create()); - resultFuture.completeExceptionally(new IllegalStateException("Query failed")); + readinessCheck + .check(requestContext) + .subscribe() + .withSubscriber(UniAssertSubscriber.create()) + .awaitFailure() + .assertFailedWith(IllegalStateException.class, "no node in the UP state"); - subscriber.awaitFailure().assertFailedWith(IllegalStateException.class, "Query failed"); verify(sessionCache, never()).evictSession(any(RequestContext.class)); } @Test - public void reactiveTimeoutBoundsSessionAcquisition() { - readinessCheck = new DatabaseReadinessCheck(sessionCache, Duration.ofMillis(20)); - when(sessionCache.getSession(requestContext)).thenReturn(Uni.createFrom().nothing()); + public void sessionAcquisitionFailureIsPropagated() { + when(sessionCache.getSession(requestContext)) + .thenReturn(Uni.createFrom().failure(new IllegalStateException("Cannot connect"))); readinessCheck .check(requestContext) .subscribe() .withSubscriber(UniAssertSubscriber.create()) .awaitFailure() - .assertFailedWith(TimeoutException.class); + .assertFailedWith(IllegalStateException.class, "Cannot connect"); - verify(session, never()).executeAsync(any(SimpleStatement.class)); + verify(session, never()).getMetadata(); verify(sessionCache, never()).evictSession(any(RequestContext.class)); } @Test - public void reactiveTimeoutBoundsAsynchronousQuery() { + public void reactiveTimeoutBoundsSessionAcquisition() { readinessCheck = new DatabaseReadinessCheck(sessionCache, Duration.ofMillis(20)); - when(sessionCache.getSession(requestContext)).thenReturn(Uni.createFrom().item(session)); - when(session.executeAsync(any(SimpleStatement.class))).thenReturn(new CompletableFuture<>()); + when(sessionCache.getSession(requestContext)).thenReturn(Uni.createFrom().nothing()); readinessCheck .check(requestContext) @@ -141,7 +138,7 @@ public void reactiveTimeoutBoundsAsynchronousQuery() { .awaitFailure() .assertFailedWith(TimeoutException.class); - verify(session).executeAsync(any(SimpleStatement.class)); + verify(session, never()).getMetadata(); verify(sessionCache, never()).evictSession(any(RequestContext.class)); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResourceTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResourceTest.java index c3db7999f8..f9bd3bfae1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResourceTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResourceTest.java @@ -9,8 +9,9 @@ import static org.mockito.Mockito.when; import com.datastax.oss.driver.api.core.CqlSession; -import com.datastax.oss.driver.api.core.cql.AsyncResultSet; -import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import com.datastax.oss.driver.api.core.metadata.Metadata; +import com.datastax.oss.driver.api.core.metadata.Node; +import com.datastax.oss.driver.api.core.metadata.NodeState; import io.quarkus.security.UnauthorizedException; import io.quarkus.test.InjectMock; import io.quarkus.test.junit.QuarkusTest; @@ -23,9 +24,10 @@ import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache; import io.stargate.sgv2.jsonapi.service.cqldriver.CqlSessionCacheSupplier; import io.stargate.sgv2.jsonapi.testresource.NoGlobalResourcesTestProfile; +import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.concurrent.CompletableFuture; +import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -73,7 +75,6 @@ public void missingTokenIsRejectedBeforeDatabaseAccess() { @ParameterizedTest @ValueSource(strings = {"", "/"}) public void astraRequestUsesResolvedTenantTokenAndSlaUserAgent(String pathSuffix) { - var resultSet = mock(AsyncResultSet.class); var capturedContext = new AtomicReference(); when(sessionCache.getSession(any(RequestContext.class))) .thenAnswer( @@ -87,8 +88,7 @@ public void astraRequestUsesResolvedTenantTokenAndSlaUserAgent(String pathSuffix context.userAgent().toString())); return Uni.createFrom().item(session); }); - when(session.executeAsync(any(SimpleStatement.class))) - .thenReturn(CompletableFuture.completedFuture(resultSet)); + stubSessionMetadata(NodeState.UP); authenticatedRequest() .when() @@ -101,6 +101,20 @@ public void astraRequestUsesResolvedTenantTokenAndSlaUserAgent(String pathSuffix .isEqualTo(new RequestContextSnapshot(TENANT_ID, REGION, TOKEN, SLA_USER_AGENT)); } + @Test + public void noUpNodeInSessionMetadataReturnsServiceUnavailable() { + when(sessionCache.getSession(any(RequestContext.class))) + .thenReturn(Uni.createFrom().item(session)); + stubSessionMetadata(NodeState.DOWN); + + authenticatedRequest() + .when() + .get(DatabaseReadinessResource.BASE_PATH) + .then() + .statusCode(503) + .body("status", equalTo("DOWN")); + } + @Test public void wrongSlaUserAgentIsRejectedBeforeDatabaseAccess() { given() @@ -209,6 +223,18 @@ public void cyclicFailureCauseReturnsServiceUnavailable() { assertThat(response).doesNotContain("first sensitive failure", "second sensitive failure"); } + private void stubSessionMetadata(NodeState... nodeStates) { + var metadata = mock(Metadata.class); + var nodes = new HashMap(); + for (NodeState nodeState : nodeStates) { + var node = mock(Node.class); + when(node.getState()).thenReturn(nodeState); + nodes.put(UUID.randomUUID(), node); + } + when(metadata.getNodes()).thenReturn(Map.copyOf(nodes)); + when(session.getMetadata()).thenReturn(metadata); + } + private io.restassured.specification.RequestSpecification authenticatedRequest() { return given() .header("Host", ASTRA_HOST) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java index 5afb578c28..56fdf3f6eb 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/SessionEvictionIntegrationTest.java @@ -52,13 +52,6 @@ protected int getCassandraCqlPort() { @Test public void testSessionEvictionOnAllNodesFailed() { - if (!executeCqlStatement( - "CREATE KEYSPACE IF NOT EXISTS datastax_sla " - + "WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}", - "CREATE TABLE IF NOT EXISTS datastax_sla.check (id text PRIMARY KEY)")) { - throw new AssertionError("Failed to provision the distributed readiness table"); - } - // 1. Insert and find initial data to ensure the database is healthy before the test insertDoc( """ diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlDriverConfigLoadEnvTests.java b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlDriverConfigLoadEnvTests.java index 09b0746339..c075828632 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlDriverConfigLoadEnvTests.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlDriverConfigLoadEnvTests.java @@ -37,7 +37,7 @@ static void initializeSessionFactory() { // Env var overrides; must be called before tests. So just constructs // a factory instance to force classloading; not used for anything CqlSessionFactory factory = - new CqlSessionFactory("test", "DC0", List.of("127.0.0.1"), 1111, () -> null); + new CqlSessionFactory("test", "DC0", List.of("127.0.0.1"), 1111, false, () -> null); } @BeforeEach From 348bd486fdfbdb285bb72a4a4d04ba7560e56618 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Mon, 10 Aug 2026 15:34:55 -0700 Subject: [PATCH 09/12] chore: tighten readiness javadoc and comments --- .../api/health/DatabaseReadinessCheck.java | 23 ++++++------------- .../api/v1/DatabaseReadinessResource.java | 14 +++++------ 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java b/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java index a83baaf7c2..a017dd4006 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java @@ -10,18 +10,12 @@ import java.util.function.Supplier; /** - * Runs the database probe exposed at {@code GET /v1/health/ready}. + * Database probe behind {@code GET /v1/health/ready}, constructed by the JAX-RS resource - not a + * CDI bean or a MicroProfile health check. * - *

The probe obtains a session through the normal session cache and inspects the driver session - * metadata, which the driver populates from {@code system.local} and {@code system.peers} and keeps - * current through node state events. The pod is ready when at least one node is {@link - * NodeState#UP}. Acquiring the session is itself part of the check: a pod that cannot connect to - * the database fails session creation and reports {@code DOWN}, and no query is issued against the - * database. - * - *

This class is constructed by the JAX-RS resource and is not a CDI bean or a MicroProfile - * health check. The caller's request context supplies the tenant, token, and User-Agent for both - * Astra and Cassandra connections. + *

Gets a session from the {@link CQLSessionCache} and succeeds when the session metadata has at + * least one {@link NodeState#UP} node. No query is issued: session creation fails if the database + * is unreachable, and the driver keeps node states current for cached sessions. */ public final class DatabaseReadinessCheck { @@ -45,10 +39,7 @@ private DatabaseReadinessCheck(Supplier sessionCacheSupplier, D this.timeout = Objects.requireNonNull(timeout, "timeout must not be null"); } - /** - * Obtains a session from the cache and fails unless the session metadata has at least one node in - * the {@link NodeState#UP} state. - */ + /** Fails unless the session metadata has at least one {@link NodeState#UP} node. */ public Uni check(RequestContext requestContext) { Objects.requireNonNull(requestContext, "requestContext must not be null"); @@ -72,7 +63,7 @@ public Uni check(RequestContext requestContext) { }) .replaceWithVoid(); }) - // Bounds asynchronous session acquisition, the metadata inspection is in-memory only. + // bounds session acquisition, the metadata inspection is in-memory .ifNoItem() .after(timeout) .fail(); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java index 078402ddf1..b9b6c56663 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java @@ -26,15 +26,13 @@ import org.jboss.resteasy.reactive.RestResponse; /** - * Authenticated database readiness endpoint registered through Quarkus JAX-RS resource discovery. + * Authenticated readiness endpoint at {@code GET /v1/health/ready}, registered through Quarkus + * JAX-RS discovery; the existing {@code /v1/*} security policy rejects requests without a token. + * Same probe for Astra and Cassandra, see {@link DatabaseReadinessCheck}. * - *

{@code GET /v1/health/ready} runs the same request-scoped probe for Astra and Cassandra: it - * obtains a session through the normal session cache and checks the driver session metadata for an - * UP node, see {@link DatabaseReadinessCheck}. A request must use the configured SLA User-Agent. A - * successful probe returns HTTP 200; invalid credentials return HTTP 401; a missing or different - * SLA User-Agent returns HTTP 403; and a session failure, timeout, or missing SLA configuration - * returns HTTP 503. The existing {@code /v1/*} security policy rejects requests without a token - * before this resource is called. + *

Requests must send the configured SLA User-Agent so probe sessions get the shorter SLA cache + * TTL. Responses: 200 UP, 401 invalid auth, 403 wrong User-Agent, 503 DOWN (session failure, + * timeout, or SLA User-Agent not configured). */ @Path(DatabaseReadinessResource.BASE_PATH) @Produces(MediaType.APPLICATION_JSON) From c220a33a65d6191cf437272f14847fbe6e3b0ff6 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Wed, 12 Aug 2026 13:59:49 -0700 Subject: [PATCH 10/12] fix: drop stale CqlSessionFactory ctor arg after removing node-metadata flag --- .../jsonapi/service/cqldriver/CqlDriverConfigLoadEnvTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlDriverConfigLoadEnvTests.java b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlDriverConfigLoadEnvTests.java index c075828632..09b0746339 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlDriverConfigLoadEnvTests.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/service/cqldriver/CqlDriverConfigLoadEnvTests.java @@ -37,7 +37,7 @@ static void initializeSessionFactory() { // Env var overrides; must be called before tests. So just constructs // a factory instance to force classloading; not used for anything CqlSessionFactory factory = - new CqlSessionFactory("test", "DC0", List.of("127.0.0.1"), 1111, false, () -> null); + new CqlSessionFactory("test", "DC0", List.of("127.0.0.1"), 1111, () -> null); } @BeforeEach From ef237d8bf8356b2ce203ff55165e4c2a2f353d45 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Wed, 12 Aug 2026 14:10:52 -0700 Subject: [PATCH 11/12] docs: tighten readiness comments and configuration docs --- CONFIGURATION.md | 93 ++++++++----------- .../api/health/DatabaseReadinessCheck.java | 12 +-- .../api/v1/DatabaseReadinessResource.java | 4 +- 3 files changed, 47 insertions(+), 62 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 0ff18f8181..d76214ceb9 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -56,60 +56,45 @@ Other Quarkus properties that are specifically relevant for the service: ### Database readiness -`GET /v1/health/ready` is an authenticated database readiness endpoint used for both Astra and -Cassandra deployments. It uses the request's tenant, `Token` header, and `User-Agent` to obtain a -session through the normal session cache. The Data API does not store separate readiness -credentials. - -The check is based on the driver's session metadata, which the driver populates from -`system.local` and `system.peers` when the session connects and keeps current through node state -events. The pod is ready when the session metadata reports at least one node in the `UP` state. No -query is issued against the database: acquiring the session is itself part of the check, because a -pod that cannot connect to the database fails session creation and reports `DOWN`. An `UP` response -therefore confirms that this pod holds a usable session with a live connection to the database. It -does not validate every tenant's credentials, quorum availability, write availability, or -cross-region availability. - -Session creation itself carries a second, independent guard: `CqlSessionFactory` rejects any newly -built session whose driver metadata is missing the `system` keyspace, so a session that connected -but could not read the schema is never cached or handed to a request. That guard is always on and -applies to all session creation, not only readiness requests. - -The two checks cover different moments, and the interaction matters for probe configuration. The -factory guard runs once, when a session is created; the readiness check runs on every probe against -whatever session is cached. Because a failed session creation surfaces as a failed session -acquisition, a probe token that authenticates but cannot read the schema tables makes this endpoint -report `DOWN` rather than an authorization error. Since readiness drives pod rotation, a probe token -whose schema-read permission is revoked will take every pod out of service. Grant the probe token -schema read access and alert on a fleet-wide `DOWN` transition, which indicates a credential problem -rather than a database outage. - -The caller must send the full User-Agent configured by -`stargate.jsonapi.operations.sla-user-agent`. The comparison is case-insensitive. Requests with a -missing or different User-Agent are rejected before accessing the session cache, and the endpoint -fails closed when the SLA User-Agent is not configured. This ensures the probe session uses the -shorter SLA session TTL instead of being treated like normal client traffic. Do not reuse the probe -credentials for normal traffic, because using the same cached session with a non-SLA User-Agent can -extend its lifetime. Astra callers must use the probe database hostname so the tenant and region are -resolved from `Host`; Cassandra ignores the tenant portion of `Host`. - -The check is fully asynchronous and has a five-second timeout. It returns HTTP 200 with -`{"status":"UP"}` when the session metadata reports an `UP` node and HTTP 503 with -`{"status":"DOWN"}` after a session failure, timeout, or missing SLA User-Agent configuration. It -returns the standard Data API error response with HTTP 401 when the `Token` header is missing or -authentication fails, and HTTP 403 when the request User-Agent does not match the configured SLA -User-Agent. Probe integrations must use the HTTP status as the readiness contract rather than -parsing the response body's `status` field alone. - -Kubernetes or an SLA checker must call each pod directly for this endpoint to control per-pod -readiness. An external request sent through a load balancer does not establish which pod is ready. -Restrict the endpoint to trusted probe traffic with deployment controls such as a NetworkPolicy, -mTLS, or an ingress ACL and rate limit. The User-Agent check is an operational guard, not an -authentication boundary. Kubernetes `httpGet` headers cannot reference a Secret, so -delivery of the probe token is intentionally outside the Data API configuration. Prefer an -external checker or a Secret-mounted file read by an `exec` probe; do not put the token literally in -the probe command or shell trace. The unauthenticated Quarkus health endpoints under the -non-application path do not include this database check. +`GET /v1/health/ready` is an authenticated readiness endpoint, the same probe for Astra and +Cassandra. It uses the request's tenant, `Token`, and `User-Agent` to get a session from the normal +session cache; there are no separate readiness credentials. + +The pod is ready when that session's driver metadata reports a node in the `UP` state. No query is +issued - getting the session is itself the check, since an unreachable database fails session +creation. `UP` means this pod holds a working session. It does not check quorum, write +availability, cross-region availability, or other tenants' credentials. + +| Status | Body | When | +|--------|------|------| +| 200 | `{"status":"UP"}` | session metadata has an `UP` node | +| 401 | Data API error | `Token` missing or authentication failed | +| 403 | empty | User-Agent does not match the configured SLA User-Agent | +| 503 | `{"status":"DOWN"}` | session failure, 5s timeout, or SLA User-Agent not configured | + +Probes must treat the HTTP status as the contract, not the body's `status` field. + +**User-Agent.** Callers must send the full `stargate.jsonapi.operations.sla-user-agent` value +(compared case-insensitively) so the probe session gets the shorter SLA cache TTL. Other +User-Agents are rejected before the session cache is touched, and the endpoint fails closed if the +SLA User-Agent is unset. Do not reuse probe credentials for normal traffic - the same cached +session used with a non-SLA User-Agent keeps the longer TTL. Astra callers must use the probe +database hostname so tenant and region resolve from `Host`; Cassandra ignores the tenant part. + +**Probe token needs schema read.** `CqlSessionFactory` independently rejects any new session whose +metadata is missing the `system` keyspace. A token that authenticates but cannot read the schema +therefore fails session creation, and this endpoint reports `DOWN` rather than an auth error. Since +readiness drives pod rotation, revoking the probe token's schema read access takes every pod out of +service. Alert on a fleet-wide `DOWN` transition - that pattern means a credential problem, not an +outage. + +**Deployment.** Probes must call each pod directly; a request through a load balancer says nothing +about which pod is ready. Restrict the endpoint to trusted probe traffic (NetworkPolicy, mTLS, or +an ingress ACL and rate limit) - the User-Agent check is an operational guard, not an +authentication boundary. Kubernetes `httpGet` headers cannot reference a Secret, so probe token +delivery is deliberately outside Data API config: prefer an external checker or a Secret-mounted +file read by an `exec` probe, and keep the token out of the probe command and shell trace. The +unauthenticated Quarkus health endpoints under the non-application path do not include this check. ## Jsonapi metering configuration diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java b/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java index a017dd4006..0a1305f82d 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java @@ -10,12 +10,12 @@ import java.util.function.Supplier; /** - * Database probe behind {@code GET /v1/health/ready}, constructed by the JAX-RS resource - not a - * CDI bean or a MicroProfile health check. + * Database probe behind {@code GET /v1/health/ready}. Constructed by the JAX-RS resource, not a CDI + * bean or MicroProfile health check. * - *

Gets a session from the {@link CQLSessionCache} and succeeds when the session metadata has at - * least one {@link NodeState#UP} node. No query is issued: session creation fails if the database - * is unreachable, and the driver keeps node states current for cached sessions. + *

Succeeds when the session from {@link CQLSessionCache} has a {@link NodeState#UP} node. No + * query is issued: an unreachable database fails session creation, and the driver keeps node states + * current on cached sessions. */ public final class DatabaseReadinessCheck { @@ -63,7 +63,7 @@ public Uni check(RequestContext requestContext) { }) .replaceWithVoid(); }) - // bounds session acquisition, the metadata inspection is in-memory + // bounds session acquisition; the metadata check itself is in-memory .ifNoItem() .after(timeout) .fail(); diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java index b9b6c56663..f7adf746b2 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java @@ -27,10 +27,10 @@ /** * Authenticated readiness endpoint at {@code GET /v1/health/ready}, registered through Quarkus - * JAX-RS discovery; the existing {@code /v1/*} security policy rejects requests without a token. + * JAX-RS discovery. The existing {@code /v1/*} security policy rejects requests without a token. * Same probe for Astra and Cassandra, see {@link DatabaseReadinessCheck}. * - *

Requests must send the configured SLA User-Agent so probe sessions get the shorter SLA cache + *

Callers must send the configured SLA User-Agent so probe sessions get the shorter SLA cache * TTL. Responses: 200 UP, 401 invalid auth, 403 wrong User-Agent, 503 DOWN (session failure, * timeout, or SLA User-Agent not configured). */ From e84e3c73f19ea43b065a5d9e5087bb3c525ef357 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Wed, 12 Aug 2026 16:43:41 -0700 Subject: [PATCH 12/12] docs: shorten readiness javadoc and configuration section --- CONFIGURATION.md | 42 +++++++------------ .../api/health/DatabaseReadinessCheck.java | 10 ++--- .../api/v1/DatabaseReadinessResource.java | 11 ++--- 3 files changed, 23 insertions(+), 40 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index d76214ceb9..b78670e2d2 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -58,12 +58,9 @@ Other Quarkus properties that are specifically relevant for the service: `GET /v1/health/ready` is an authenticated readiness endpoint, the same probe for Astra and Cassandra. It uses the request's tenant, `Token`, and `User-Agent` to get a session from the normal -session cache; there are no separate readiness credentials. - -The pod is ready when that session's driver metadata reports a node in the `UP` state. No query is -issued - getting the session is itself the check, since an unreachable database fails session -creation. `UP` means this pod holds a working session. It does not check quorum, write -availability, cross-region availability, or other tenants' credentials. +session cache, and reports ready when that session's driver metadata has a node in the `UP` state. +No query is issued - getting the session is itself the check. `UP` means this pod holds a working +session, nothing more; it says nothing about quorum, write availability, or other tenants. | Status | Body | When | |--------|------|------| @@ -74,27 +71,18 @@ availability, cross-region availability, or other tenants' credentials. Probes must treat the HTTP status as the contract, not the body's `status` field. -**User-Agent.** Callers must send the full `stargate.jsonapi.operations.sla-user-agent` value -(compared case-insensitively) so the probe session gets the shorter SLA cache TTL. Other -User-Agents are rejected before the session cache is touched, and the endpoint fails closed if the -SLA User-Agent is unset. Do not reuse probe credentials for normal traffic - the same cached -session used with a non-SLA User-Agent keeps the longer TTL. Astra callers must use the probe -database hostname so tenant and region resolve from `Host`; Cassandra ignores the tenant part. - -**Probe token needs schema read.** `CqlSessionFactory` independently rejects any new session whose -metadata is missing the `system` keyspace. A token that authenticates but cannot read the schema -therefore fails session creation, and this endpoint reports `DOWN` rather than an auth error. Since -readiness drives pod rotation, revoking the probe token's schema read access takes every pod out of -service. Alert on a fleet-wide `DOWN` transition - that pattern means a credential problem, not an -outage. - -**Deployment.** Probes must call each pod directly; a request through a load balancer says nothing -about which pod is ready. Restrict the endpoint to trusted probe traffic (NetworkPolicy, mTLS, or -an ingress ACL and rate limit) - the User-Agent check is an operational guard, not an -authentication boundary. Kubernetes `httpGet` headers cannot reference a Secret, so probe token -delivery is deliberately outside Data API config: prefer an external checker or a Secret-mounted -file read by an `exec` probe, and keep the token out of the probe command and shell trace. The -unauthenticated Quarkus health endpoints under the non-application path do not include this check. +- **User-Agent** - callers must send the full `stargate.jsonapi.operations.sla-user-agent` value so + the probe session gets the shorter SLA cache TTL; anything else is rejected before the session + cache is touched, and the endpoint fails closed when that setting is unset. Astra callers must + use the probe database hostname so tenant and region resolve from `Host`. +- **Probe token needs schema read** - `CqlSessionFactory` rejects any new session missing the + `system` keyspace, so a token that authenticates but cannot read the schema reports `DOWN` rather + than an auth error. Readiness drives pod rotation, so a fleet-wide `DOWN` means a credential + problem, not an outage. +- **Deployment** - probe each pod directly and restrict the endpoint to trusted traffic; the + User-Agent check is an operational guard, not an authentication boundary. Kubernetes `httpGet` + headers cannot reference a Secret, so deliver the probe token outside Data API config - an + external checker, or a Secret-mounted file read by an `exec` probe. ## Jsonapi metering configuration diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java b/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java index 0a1305f82d..d9b18b22a5 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/health/DatabaseReadinessCheck.java @@ -10,12 +10,10 @@ import java.util.function.Supplier; /** - * Database probe behind {@code GET /v1/health/ready}. Constructed by the JAX-RS resource, not a CDI - * bean or MicroProfile health check. - * - *

Succeeds when the session from {@link CQLSessionCache} has a {@link NodeState#UP} node. No - * query is issued: an unreachable database fails session creation, and the driver keeps node states - * current on cached sessions. + * Database probe behind {@code GET /v1/health/ready}, constructed by the JAX-RS resource rather + * than as a CDI bean or MicroProfile health check. Succeeds when the session from {@link + * CQLSessionCache} has a {@link NodeState#UP} node; no query is issued because an unreachable + * database already fails session creation. */ public final class DatabaseReadinessCheck { diff --git a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java index f7adf746b2..1fce750461 100644 --- a/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java +++ b/src/main/java/io/stargate/sgv2/jsonapi/api/v1/DatabaseReadinessResource.java @@ -26,13 +26,10 @@ import org.jboss.resteasy.reactive.RestResponse; /** - * Authenticated readiness endpoint at {@code GET /v1/health/ready}, registered through Quarkus - * JAX-RS discovery. The existing {@code /v1/*} security policy rejects requests without a token. - * Same probe for Astra and Cassandra, see {@link DatabaseReadinessCheck}. - * - *

Callers must send the configured SLA User-Agent so probe sessions get the shorter SLA cache - * TTL. Responses: 200 UP, 401 invalid auth, 403 wrong User-Agent, 503 DOWN (session failure, - * timeout, or SLA User-Agent not configured). + * Authenticated readiness endpoint at {@code GET /v1/health/ready}, the same probe for Astra and + * Cassandra, see {@link DatabaseReadinessCheck}. The {@code /v1/*} security policy rejects requests + * without a token, and callers must send the configured SLA User-Agent so probe sessions get the + * shorter SLA cache TTL. */ @Path(DatabaseReadinessResource.BASE_PATH) @Produces(MediaType.APPLICATION_JSON)