Add Alternator vector search support

.github/workflows/continuous-integration.yml

@@ -92,6 +92,7 @@ jobs:
         run: make cert-cache-save
 
       - name: Run integration tests and demo apps
+        timeout-minutes: 15
         run: make test-all
 
       - name: Upload integration test results

README.md

@@ -973,3 +973,267 @@ Key route affinity may not be beneficial for:
 - Read-heavy workloads (reads don't use Paxos)
 - Workloads with uniformly distributed writes across many keys
 - Async clients where partition-key names cannot be pre-configured
+
+### Vector Search (Alternator extension)
+
+Alternator extends the DynamoDB API with **vector indexes** and **vector similarity search**.
+These features are not available on AWS DynamoDB, so the standard AWS SDK for Java has no
+knowledge of the new parameters (`VectorIndexes`, `VectorSearch`, `FLOAT32VECTOR`, etc.).
+
+This library bridges the gap without modifying the SDK source. It uses an
+`ExecutionInterceptor` that intercepts the serialised JSON body of each request before
+it is sent, and of each response before the SDK parses it into Java objects:
+- **Requests** — extra parameters are injected into the JSON body before transmission.
+- **Responses** — extra fields are extracted from the JSON body before the SDK discards them.
+
+#### Setup
+
+`VectorSearchInterceptor` is registered automatically by `AlternatorDynamoDbClient` and
+`AlternatorDynamoDbAsyncClient`. No extra configuration is needed:
+
+```java
+DynamoDbClient client = AlternatorDynamoDbClient.builder()
+    .endpointOverride(URI.create("http://localhost:8000"))
+    .credentialsProvider(myCredentials)
+    .build();
+```
+
+If you use the plain `DynamoDbClient.builder()` / `DynamoDbAsyncClient.builder()` directly
+(without the Alternator builder), register the interceptor manually:
+
+```java
+import com.scylladb.alternator.vectorsearch.VectorSearchInterceptor;
+
+DynamoDbClient client = DynamoDbClient.builder()
+    .endpointOverride(URI.create("http://localhost:8000"))
+    .credentialsProvider(myCredentials)
+    .overrideConfiguration(c -> c.addExecutionInterceptor(VectorSearchInterceptor.INSTANCE))
+    .build();
+```
+
+#### CreateTable with a vector index
+
+```java
+import com.scylladb.alternator.vectorsearch.*;
+
+VectorIndex vi = VectorIndex.builder()
+    .indexName("embedding-index")
+    .vectorAttribute(VectorAttribute.builder()
+        .attributeName("embedding")
+        .dimensions(128)
+        .build())
+    .similarityFunction("COSINE")   // optional; "DOT_PRODUCT" and "EUCLIDEAN" are also supported
+    .build();
+
+CreateTableRequest base = CreateTableRequest.builder()
+    .tableName("items")
+    .keySchema(KeySchemaElement.builder().attributeName("id").keyType(KeyType.HASH).build())
+    .attributeDefinitions(
+        AttributeDefinition.builder().attributeName("id").attributeType(ScalarAttributeType.S).build())
+    .billingMode(BillingMode.PAY_PER_REQUEST)
+    .build();
+
+VectorSearchSupport.CreateTableWithVectorIndexes result =
+    VectorSearchSupport.createTable(client, base, List.of(vi));
+
+for (VectorIndex returned : result.vectorIndexes()) {
+    System.out.printf("index=%s status=%s%n",
+        returned.indexName(), returned.indexStatus());
+}
+```
+
+Async clients can retrieve the same response metadata:
+
+```java
+CompletableFuture<VectorSearchSupport.CreateTableWithVectorIndexes> result =
+    VectorSearchSupport.createTableAsync(asyncClient, base, List.of(vi));
+```
+
+You can use the lower-level request helper if you do not need the returned vector-index metadata
+and want to call `client.createTable()` yourself:
+
+```java
+client.createTable(VectorSearchSupport.withVectorIndexes(base, List.of(vi)));
+```
+
+#### UpdateTable — adding or removing a vector index
+
+Each `UpdateTable` request accepts exactly one vector index update. Adding and removing
+indexes therefore require separate requests:
+
+```java
+VectorIndexUpdate addIndex = VectorIndexUpdate.builder()
+    .create(CreateVectorIndexAction.builder()
+        .indexName("new-index")
+        .vectorAttribute(VectorAttribute.builder()
+            .attributeName("embedding")
+            .dimensions(64)
+            .build())
+        .build())
+    .build();
+
+VectorIndexUpdate removeIndex = VectorIndexUpdate.builder()
+    .delete(DeleteVectorIndexAction.builder()
+        .indexName("old-index")
+        .build())
+    .build();
+
+VectorSearchSupport.updateTable(client,
+    UpdateTableRequest.builder().tableName("items").build(),
+    List.of(addIndex));
+
+// After the first update has completed and the table is ACTIVE:
+VectorSearchSupport.updateTable(client,
+    UpdateTableRequest.builder().tableName("items").build(),
+    List.of(removeIndex));
+```
+
+You can also use the lower-level helper if you want to call `client.updateTable()` yourself:
+
+```java
+client.updateTable(VectorSearchSupport.withVectorIndexUpdates(
+    UpdateTableRequest.builder().tableName("items").build(),
+    List.of(addIndex)));
+
+// After the first update has completed and the table is ACTIVE:
+client.updateTable(VectorSearchSupport.withVectorIndexUpdates(
+    UpdateTableRequest.builder().tableName("items").build(),
+    List.of(removeIndex)));
+```
+
+#### DescribeTable — reading vector index metadata
+
+```java
+VectorSearchSupport.DescribeTableWithVectorIndexes result =
+    VectorSearchSupport.describeTable(client,
+        DescribeTableRequest.builder().tableName("items").build());
+
+for (VectorIndex vi : result.vectorIndexes()) {
+    System.out.printf("index=%s attribute=%s dimensions=%d status=%s%n",
+        vi.indexName(),
+        vi.vectorAttribute().attributeName(),
+        vi.vectorAttribute().dimensions(),
+        vi.indexStatus());
+}
+```
+
+Async clients can retrieve the same metadata without losing the extension field:
+
+```java
+CompletableFuture<VectorSearchSupport.DescribeTableWithVectorIndexes> result =
+    VectorSearchSupport.describeTableAsync(asyncClient,
+        DescribeTableRequest.builder().tableName("items").build());
+```
+
+#### Writing items with the optimized FLOAT32VECTOR type
+
+Alternator stores vectors in a compact binary format called `FLOAT32VECTOR`. This is
+significantly more efficient than the standard DynamoDB list-of-numbers encoding (`L`)
+and is recommended for use with a vector index.
+
+Use `Float32Vector.toAttributeValue(float...)` to create the attribute value. The
+interceptor automatically converts it to `{"FLOAT32VECTOR": [...]}` in the JSON body:
+
+```java
+import com.scylladb.alternator.vectorsearch.Float32Vector;
+
+Map<String, AttributeValue> item = new HashMap<>();
+item.put("id", AttributeValue.fromS("item-1"));
+item.put("embedding", Float32Vector.toAttributeValue(0.1f, 0.2f, 0.3f, ...));
+
+client.putItem(PutItemRequest.builder().tableName("items").item(item).build());
+```
+
+This works transparently for all write operations: `PutItem`, `UpdateItem`
+(`ExpressionAttributeValues`), and `BatchWriteItem`.
+
+#### Reading FLOAT32VECTOR attributes back
+
+When Alternator returns a `FLOAT32VECTOR` attribute, the interceptor converts it
+transparently to a magic-prefixed binary marker. This retains the optimized type
+even though the standard SDK does not know about `FLOAT32VECTOR`. Identify and
+decode it with `Float32Vector`:
+
+```java
+AttributeValue av = resp.item().get("embedding");
+if (Float32Vector.isFloat32Vector(av)) {
+    float[] values = Float32Vector.toFloats(av);
+}
+```
+
+Returned vector markers can be passed directly back into a write. The interceptor
+recognizes them and re-emits `FLOAT32VECTOR`, so read-modify-write and item-copy
+operations retain the compact storage type:
+
+```java
+Map<String, AttributeValue> item = new HashMap<>(response.item());
+item.put("description", AttributeValue.fromS("updated"));
+client.putItem(PutItemRequest.builder().tableName("items").item(item).build());
+```
+
+The `Float32Vector.toAttributeValue(List<AttributeValue>)` overload remains available
+for explicitly converting an ordinary DynamoDB `L` vector to `FLOAT32VECTOR`.
+
+#### Vector similarity search (Query)
+
+Standard `QueryRequest` parameters work alongside `VectorSearch`: `filterExpression`,
+`select`, `projectionExpression`, and `expressionAttributeValues` are supported. The
+`limit` parameter is **required** for vector search queries and specifies how many
+nearest neighbours to return. `keyConditionExpression` and `exclusiveStartKey`
+(pagination) are not supported.
+
+```java
+VectorSearch vs = VectorSearch.builder()
+    .queryVector(0.1f, 0.2f, 0.3f, ...)         // sent as FLOAT32VECTOR
+    .returnScores(true)                         // optional; include similarity scores
+    .build();
+
+VectorQueryResult result = VectorSearchSupport.query(client,
+    QueryRequest.builder()
+        .tableName("items")
+        .indexName("embedding-index")
+        .limit(10)
+        .build(),
+    vs);
+
+for (int i = 0; i < result.items().size(); i++) {
+    System.out.printf("item=%s score=%.4f%n",
+        result.items().get(i).get("id").s(),
+        result.scores().get(i));
+}
+```
+
+For the async client:
+
+```java
+CompletableFuture<VectorQueryResult> future =
+    VectorSearchSupport.queryAsync(asyncClient, queryRequest, vs);
+```
+
+You can also pass the query vector as an `AttributeValue` (standard DynamoDB list type)
+instead of a `float[]`, which is useful when the items were stored without the
+`FLOAT32VECTOR` optimisation:
+
+```java
+VectorSearch vs = VectorSearch.builder()
+    .queryVector(AttributeValue.fromL(Arrays.asList(
+        AttributeValue.fromN("0.1"),
+        AttributeValue.fromN("0.2"),
+        AttributeValue.fromN("0.3"))))
+    .build();
+```
+
+#### Summary of the `vectorsearch` package
+
+| Class | Purpose |
+|---|---|
+| `VectorSearchInterceptor` | Core interceptor — register once on the client |
+| `VectorSearchSupport` | Convenience facade for all operations |
+| `VectorIndex` | Descriptor for a vector index (create/describe) |
+| `VectorAttribute` | Attribute name + dimensionality for a `VectorIndex` |
+| `VectorSearch` | Query parameters: query vector + optional score request |
+| `VectorQueryResult` | Wraps `QueryResponse` and adds `scores()` |
+| `VectorIndexUpdate` | A single create/delete change for `UpdateTable` |
+| `CreateVectorIndexAction` | Create action inside a `VectorIndexUpdate` |
+| `DeleteVectorIndexAction` | Delete action inside a `VectorIndexUpdate` |
+| `Float32Vector` | Encode/decode `float...` as the compact `FLOAT32VECTOR` type |

pom.xml

@@ -23,10 +23,18 @@
             <dependency>
                 <groupId>software.amazon.awssdk</groupId>
                 <artifactId>bom</artifactId>
-                <version>2.42.2</version>
+                <version>2.48.0</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
+            <!-- jackson-databind is a transitive dependency of the AWS SDK but is not
+                 exported as a managed version in the SDK BOM. Pin it here centrally
+                 so the version appears in one place and can be overridden by a parent POM. -->
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-databind</artifactId>
+                <version>2.22.1</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
     <dependencies>
@@ -51,6 +59,14 @@
             <artifactId>aws-crt-client</artifactId>
             <scope>provided</scope>
         </dependency>
+        <!-- Jackson Databind is used by VectorSearchInterceptor for JSON manipulation.
+             Publish it transitively so clients can construct the auto-configured
+             AlternatorDynamoDbClient without declaring Jackson themselves. Version is
+             managed in <dependencyManagement> above. -->
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
         <dependency>
 			<groupId>net.sourceforge.argparse4j</groupId>
 			<artifactId>argparse4j</artifactId>

src/integration-test/java/com/scylladb/alternator/AlternatorDynamoDbAsyncClientIT.java

@@ -365,7 +365,8 @@ public void beforeTransmission(
                   .build())
           .get();
     } catch (Exception e) {
-      // Table doesn't exist - that's expected, we just want to verify the request was compressed
+      // Scylla 2025.1 does not decode gzip request bodies. This test verifies the client-side
+      // transport headers; byte-for-byte compression is covered by GzipRequestInterceptorTest.
     }
 
     assertTrue(
@@ -737,7 +738,8 @@ public void beforeTransmission(
                   .build())
           .get();
     } catch (Exception e) {
-      // Expected - table doesn't exist
+      // Scylla 2025.1 does not decode gzip request bodies. The assertions below verify that header
+      // optimization preserves the compression headers.
     }
 
     // Verify both features work together

src/integration-test/java/com/scylladb/alternator/AlternatorDynamoDbClientIT.java

@@ -353,8 +353,9 @@ public void beforeTransmission(
                       "ID", AttributeValue.builder().s("compression-test").build(),
                       "LargeData", AttributeValue.builder().s(largeValue.toString()).build()))
               .build());
-    } catch (ResourceNotFoundException e) {
-      // Table doesn't exist - that's expected, we just want to verify the request was compressed
+    } catch (DynamoDbException e) {
+      // Scylla 2025.1 does not decode gzip request bodies. This test verifies the client-side
+      // transport headers; byte-for-byte compression is covered by GzipRequestInterceptorTest.
     }
 
     assertTrue(
@@ -724,8 +725,9 @@ public void beforeTransmission(
                       "ID", AttributeValue.builder().s("test").build(),
                       "LargeData", AttributeValue.builder().s(largeValue.toString()).build()))
               .build());
-    } catch (ResourceNotFoundException e) {
-      // Expected - table doesn't exist
+    } catch (DynamoDbException e) {
+      // Scylla 2025.1 does not decode gzip request bodies. The assertions below verify that header
+      // optimization preserves the compression headers.
     }
 
     // Verify both features work together

src/integration-test/java/com/scylladb/alternator/HttpClientImplementationAsyncIT.java

@@ -436,7 +436,8 @@ public void beforeTransmission(
                   .build())
           .get();
     } catch (Exception e) {
-      // Expected — table doesn't exist
+      // Scylla 2025.1 does not decode gzip request bodies. This test verifies that each HTTP client
+      // receives the compressed request and its transport headers.
     } finally {
       client.close();
     }
@@ -483,7 +484,7 @@ public void beforeTransmission(
                   .build())
           .get();
     } catch (Exception e) {
-      // Expected
+      // Scylla 2025.1 does not decode gzip request bodies. Header behavior is asserted below.
     } finally {
       client.close();
     }

src/integration-test/java/com/scylladb/alternator/HttpClientImplementationSyncIT.java

@@ -434,8 +434,9 @@ public void beforeTransmission(
                   "pk", AttributeValue.builder().s("k").build(),
                   "data", AttributeValue.builder().s(largePayload()).build()))
               .build());
-    } catch (ResourceNotFoundException e) {
-      // Expected — table doesn't exist
+    } catch (DynamoDbException e) {
+      // Scylla 2025.1 does not decode gzip request bodies. This test verifies that each HTTP client
+      // receives the compressed request and its transport headers.
     } finally {
       client.close();
     }
@@ -479,8 +480,8 @@ public void beforeTransmission(
                   "pk", AttributeValue.builder().s("k").build(),
                   "data", AttributeValue.builder().s(largePayload()).build()))
               .build());
-    } catch (ResourceNotFoundException e) {
-      // Expected
+    } catch (DynamoDbException e) {
+      // Scylla 2025.1 does not decode gzip request bodies. Header behavior is asserted below.
     } finally {
       client.close();
     }