From 49a35f17a1277de71c5f7e237754a88a540a001f Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Wed, 10 Dec 2025 15:08:09 +0530 Subject: [PATCH 1/9] Use ARRAY instead of IN queries for perf --- .../documentstore/DocStoreQueryV1Test.java | 237 +++++++++++++++++- ...gresInRelationalFilterParserJsonArray.java | 4 +- ...InRelationalFilterParserJsonPrimitive.java | 11 +- ...resInRelationalFilterParserArrayField.java | 6 +- ...esInRelationalFilterParserScalarField.java | 4 +- 5 files changed, 238 insertions(+), 24 deletions(-) diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index 3bcaa27b..8ab263a5 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -68,6 +68,7 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -3967,18 +3968,18 @@ void testNumericAggregations(String dataStoreName) throws IOException { JsonNode json = new ObjectMapper().readTree(aggDoc.toJson()); // Validate aggregation results - assertTrue(json.get("sum").asDouble() > 0, "SUM should be positive"); - assertTrue(json.get("avg").asDouble() > 0, "AVG should be positive"); - assertTrue(json.get("min").asDouble() > 0, "MIN should be positive"); - assertTrue(json.get("max").asDouble() > 0, "MAX should be positive"); - assertEquals(10, json.get("count").asInt(), "COUNT should be 10"); + assertTrue(json.get("sum").asDouble() > 0); + assertTrue(json.get("avg").asDouble() > 0); + assertTrue(json.get("min").asDouble() > 0); + assertTrue(json.get("max").asDouble() > 0); + assertEquals(10, json.get("count").asInt()); // Verify MIN <= AVG <= MAX double min = json.get("min").asDouble(); double avg = json.get("avg").asDouble(); double max = json.get("max").asDouble(); - assertTrue(min <= avg, "MIN should be <= AVG"); - assertTrue(avg <= max, "AVG should be <= MAX"); + assertTrue(min <= avg); + assertTrue(avg <= max); // Test GROUP BY with aggregations Query groupAggQuery = @@ -4049,11 +4050,11 @@ void testNullHandling(String dataStoreName) throws IOException { int countPrice = json.get("count_price").asInt(); int countAll = json.get("count_all").asInt(); - assertEquals(10, countItem, "COUNT(item) should be 10"); - assertEquals(10, countPrice, "COUNT(price) should be 10"); - assertEquals(10, countAll, "COUNT(*) should be 10"); - assertEquals(countItem, countAll, "COUNT(item) should equal COUNT(*) when no NULLs"); - assertEquals(countPrice, countAll, "COUNT(price) should equal COUNT(*) when no NULLs"); + assertEquals(10, countItem); + assertEquals(10, countPrice); + assertEquals(10, countAll); + assertEquals(countItem, countAll); + assertEquals(countPrice, countAll); // Test 3: Test NULL equality filter returns empty result Query nullEqualQuery = @@ -4066,6 +4067,218 @@ void testNullHandling(String dataStoreName) throws IOException { long nullCount = flatCollection.count(nullEqualQuery); assertEquals(0, nullCount); } + + @ParameterizedTest + @ArgumentsSource(PostgresProvider.class) + void testInOperatorOnScalarFields(String dataStoreName) throws JsonProcessingException { + Collection flatCollection = getFlatCollection(dataStoreName); + + // Test 1: IN operator on TEXT field (item column) + Query textInQuery = + Query.builder() + .addSelection(IdentifierExpression.of("item")) + .addSelection(IdentifierExpression.of("price")) + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("item"), + IN, + ConstantExpression.ofStrings(List.of("Soap", "Mirror", "Comb")))) + .build(); + + Iterator textResults = flatCollection.find(textInQuery); + Set foundItems = new HashSet<>(); + int textCount = 0; + while (textResults.hasNext()) { + Document doc = textResults.next(); + JsonNode json = new ObjectMapper().readTree(doc.toJson()); + textCount++; + foundItems.add(json.get("item").asText()); + } + assertEquals(6, textCount); + assertTrue(foundItems.contains("Soap")); + assertTrue(foundItems.contains("Mirror")); + assertTrue(foundItems.contains("Comb")); + assertEquals(3, foundItems.size()); + + // Test 2: IN operator on INTEGER field (price column) + Query intInQuery = + Query.builder() + .addSelection(IdentifierExpression.of("item")) + .addSelection(IdentifierExpression.of("price")) + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("price"), + IN, + ConstantExpression.ofNumbers(List.of(5, 10, 15)))) + .build(); + + Iterator intResults = flatCollection.find(intInQuery); + Set foundPrices = new HashSet<>(); + int intCount = 0; + while (intResults.hasNext()) { + Document doc = intResults.next(); + JsonNode json = new ObjectMapper().readTree(doc.toJson()); + intCount++; + foundPrices.add(json.get("price").asInt()); + } + assertTrue(intCount > 0); + assertTrue(foundPrices.stream().allMatch(p -> p == 5 || p == 10 || p == 15)); + + // Test 3: IN operator on BOOLEAN field (in_stock column) + Query boolInQuery = + Query.builder() + .addSelection(IdentifierExpression.of("item")) + .addSelection(IdentifierExpression.of("in_stock")) + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("in_stock"), + IN, + ConstantExpression.ofBooleans(List.of(true)))) + .build(); + + Iterator boolResults = flatCollection.find(boolInQuery); + int boolCount = 0; + while (boolResults.hasNext()) { + Document doc = boolResults.next(); + JsonNode json = new ObjectMapper().readTree(doc.toJson()); + boolCount++; + assertTrue(json.get("in_stock").asBoolean()); + } + assertTrue(boolCount > 0); + + // Test 4: IN operator with single value + Query singleValueInQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("item"), + IN, + ConstantExpression.ofStrings(List.of("Soap")))) + .build(); + + long singleValueCount = flatCollection.count(singleValueInQuery); + assertEquals(3, singleValueCount); + + // Test 5: IN operator with no matches + Query noMatchInQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("item"), + IN, + ConstantExpression.ofStrings(List.of("NonExistent", "AlsoNotFound")))) + .build(); + + long noMatchCount = flatCollection.count(noMatchInQuery); + assertEquals(0, noMatchCount); + + // Test 6: IN operator with large list + List largeList = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + largeList.add("Item" + i); + } + largeList.add("Soap"); + + Query largeListInQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("item"), IN, ConstantExpression.ofStrings(largeList))) + .build(); + + long largeListCount = flatCollection.count(largeListInQuery); + assertEquals(3, largeListCount); + } + + @ParameterizedTest + @ArgumentsSource(PostgresProvider.class) + void testNotInOperatorOnScalarFields(String dataStoreName) { + Collection flatCollection = getFlatCollection(dataStoreName); + + // Test 1: NOT IN operator on TEXT field + Query textNotInQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("item"), + NOT_IN, + ConstantExpression.ofStrings(List.of("Soap", "Mirror")))) + .build(); + + long textNotInCount = flatCollection.count(textNotInQuery); + assertEquals(6, textNotInCount); + + // Test 2: NOT IN operator on INTEGER field + Query intNotInQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("price"), + NOT_IN, + ConstantExpression.ofNumbers(List.of(5, 10)))) + .build(); + + long intNotInCount = flatCollection.count(intNotInQuery); + assertTrue(intNotInCount > 0); + + // Test 3: NOT IN operator on BOOLEAN field + Query boolNotInQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("in_stock"), + NOT_IN, + ConstantExpression.ofBooleans(List.of(false)))) + .build(); + + long boolNotInCount = flatCollection.count(boolNotInQuery); + assertTrue(boolNotInCount > 0); + + // Test 4: NOT IN with single value + Query singleNotInQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("item"), + NOT_IN, + ConstantExpression.ofStrings(List.of("Soap")))) + .build(); + + long singleNotInCount = flatCollection.count(singleNotInQuery); + assertEquals(7, singleNotInCount); + + // Test 5: NOT IN with values that don't exist (should return all records) + Query noMatchNotInQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("item"), + NOT_IN, + ConstantExpression.ofStrings(List.of("NonExistent", "AlsoNotFound")))) + .build(); + + long noMatchNotInCount = flatCollection.count(noMatchNotInQuery); + assertEquals(10, noMatchNotInCount); + + // Test 6: NOT IN with large list + List largeList = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + largeList.add("Item" + i); + } + largeList.add("Soap"); + + Query largeListNotInQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("item"), + NOT_IN, + ConstantExpression.ofStrings(largeList))) + .build(); + + long largeListNotInCount = flatCollection.count(largeListNotInQuery); + assertEquals(7, largeListNotInCount); + } } @Nested diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonArray.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonArray.java index 23391791..798a26f6 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonArray.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonArray.java @@ -61,7 +61,7 @@ public String parse( /** * Generates SQL for scalar IN operator (used when JSONB array field has been unnested). Example: - * "props_dot_source-loc" IN (?::jsonb, ?::jsonb) + * "props_dot_source-loc" = ANY(ARRAY[?::jsonb, ?::jsonb]) * *

Note: After unnesting with jsonb_array_elements(), each row contains a JSONB scalar value. * We cast the parameters to jsonb for direct JSONB-to-JSONB comparison, which works for all JSONB @@ -86,7 +86,7 @@ private String prepareFilterStringForScalarInOperator( .collect(Collectors.joining(", ")); // Direct JSONB comparison - no text conversion needed - return String.format("%s IN (%s)", parsedLhs, placeholders); + return String.format("%s = ANY(ARRAY[%s])", parsedLhs, placeholders); } /** diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java index f381e745..9a2a2686 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java @@ -11,12 +11,13 @@ * Optimized parser for IN operations on JSON primitive fields (string, number, boolean) with proper * type casting. * - *

Generates efficient SQL using {@code ->>} operator with appropriate PostgreSQL casting: + *

Generates efficient SQL using {@code ->>} operator with {@code = ANY(ARRAY[])} and appropriate + * PostgreSQL casting: * *

    - *
  • STRING: {@code "document" ->> 'item' IN ('Soap', 'Shampoo')} - *
  • NUMBER: {@code CAST("document" ->> 'price' AS NUMERIC) IN (10, 20)} - *
  • BOOLEAN: {@code CAST("document" ->> 'active' AS BOOLEAN) IN (true, false)} + *
  • STRING: {@code "document" ->> 'item' = ANY(ARRAY['Soap', 'Shampoo'])} + *
  • NUMBER: {@code CAST("document" ->> 'price' AS NUMERIC) = ANY(ARRAY[10, 20])} + *
  • BOOLEAN: {@code CAST("document" ->> 'active' AS BOOLEAN) = ANY(ARRAY[true, false])} *
* *

This is much more efficient than the defensive approach that checks both array and scalar @@ -79,6 +80,6 @@ private String prepareFilterStringForInOperator( } // STRING or null fieldType: no casting needed - return String.format("%s IN (%s)", lhsWithCast, placeholders); + return String.format("%s = ANY(ARRAY[%s])", lhsWithCast, placeholders); } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java index a120ac14..24f37308 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java @@ -49,7 +49,7 @@ public String parse( /** * Generates SQL for scalar IN operator (used when array field has been unnested). Example: - * "tags_unnested" IN (?, ?, ?) + * "tags_unnested" = ANY(ARRAY[?, ?, ?]) */ private String prepareFilterStringForScalarInOperator( final String parsedLhs, @@ -65,8 +65,8 @@ private String prepareFilterStringForScalarInOperator( }) .collect(Collectors.joining(", ")); - // Scalar IN operator for unnested array elements - return String.format("%s IN (%s)", parsedLhs, placeholders); + // Optimized scalar IN operator for unnested array elements + return String.format("%s = ANY(ARRAY[%s])", parsedLhs, placeholders); } /** diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java index 274c12a8..bce4e73e 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java @@ -9,7 +9,7 @@ /** * Implementation of PostgresInRelationalFilterParserInterface for handling IN operations on - * first-class fields (non-JSON columns), using the standard IN clause syntax. + * first-class fields (non-JSON columns), using the optimized = ANY(ARRAY[]) syntax. */ public class PostgresInRelationalFilterParserScalarField implements PostgresInRelationalFilterParserInterface { @@ -38,6 +38,6 @@ private String prepareFilterStringForInOperator( }) .collect(Collectors.joining(", ")); - return String.format("%s IN (%s)", parsedLhs, placeholders); + return String.format("%s = ANY(ARRAY[%s])", parsedLhs, placeholders); } } From 2c3e834b646c1b9aabaec980c13a358a89dfeee1 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Wed, 10 Dec 2025 15:21:56 +0530 Subject: [PATCH 2/9] Fixed PostgresQueryParserTest --- .../postgres/query/v1/PostgresQueryParserTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java index 9c24380c..f698de3a 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java @@ -684,7 +684,7 @@ void testFindWithSortingAndPagination() { + "\"quantity\" AS \"quantity\", " + "\"date\" AS \"date\" " + "FROM \"testCollection\" " - + "WHERE \"item\" IN (?, ?, ?, ?) " + + "WHERE \"item\" = ANY(ARRAY[?, ?, ?, ?]) " + "ORDER BY \"quantity\" DESC NULLS LAST,\"item\" ASC NULLS FIRST " + "OFFSET ? LIMIT ?", postgresQueryParser.parse()); @@ -1506,7 +1506,7 @@ void testNotInWithFlatCollectionNonJsonField() { String sql = postgresQueryParser.parse(); assertEquals( - "SELECT * FROM \"testCollection\" WHERE \"category\" IS NULL OR NOT (\"category\" IN (?, ?))", + "SELECT * FROM \"testCollection\" WHERE \"category\" IS NULL OR NOT (\"category\" = ANY(ARRAY[?, ?]))", sql); Params params = postgresQueryParser.getParamsBuilder().build(); From 712b605afee76d9029cc0c62a3f06d1cbbf2e882 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Thu, 11 Dec 2025 15:06:54 +0530 Subject: [PATCH 3/9] Use array parms correctly --- .../core/documentstore/postgres/Params.java | 18 +++ ...InRelationalFilterParserJsonPrimitive.java | 16 +- ...resInRelationalFilterParserArrayField.java | 82 +++++----- ...esInRelationalFilterParserScalarField.java | 15 +- .../postgres/utils/PostgresUtils.java | 36 ++++- .../query/v1/PostgresQueryParserTest.java | 143 +++++++++++++++++- 6 files changed, 247 insertions(+), 63 deletions(-) diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/Params.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/Params.java index 6bc61a26..e5c2dc36 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/Params.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/Params.java @@ -2,6 +2,7 @@ import java.util.HashMap; import java.util.Map; +import lombok.Getter; /** * Holds the params that need to be set in the PreparedStatement for constructing the final SQL @@ -29,6 +30,18 @@ public static Builder newBuilder() { return new Builder(); } + /** Wrapper class to hold array parameter metadata for PostgreSQL array binding */ + @Getter + public static class ArrayParam { + private final Object[] values; + private final String sqlType; + + public ArrayParam(Object[] values, String sqlType) { + this.values = values; + this.sqlType = sqlType; + } + } + public static class Builder { private int nextIndex; @@ -44,6 +57,11 @@ public Builder addObjectParam(Object paramValue) { return this; } + public Builder addArrayParam(Object[] values, String sqlType) { + objectParams.put(nextIndex++, new ArrayParam(values, sqlType)); + return this; + } + public Params build() { return new Params(objectParams); } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java index 9a2a2686..81835c67 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java @@ -1,11 +1,11 @@ package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter; -import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.hypertrace.core.documentstore.expression.impl.JsonFieldType; import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; import org.hypertrace.core.documentstore.postgres.Params; +import org.hypertrace.core.documentstore.postgres.utils.PostgresUtils; /** * Optimized parser for IN operations on JSON primitive fields (string, number, boolean) with proper @@ -62,14 +62,10 @@ private String prepareFilterStringForInOperator( final JsonFieldType fieldType, final Params.Builder paramsBuilder) { - String placeholders = - StreamSupport.stream(parsedRhs.spliterator(), false) - .map( - value -> { - paramsBuilder.addObjectParam(value); - return "?"; - }) - .collect(Collectors.joining(", ")); + Object[] values = StreamSupport.stream(parsedRhs.spliterator(), false).toArray(); + + // Add as single array parameter + paramsBuilder.addArrayParam(values, PostgresUtils.inferSqlTypeFromValue(values)); // Apply appropriate casting based on field type String lhsWithCast = parsedLhs; @@ -80,6 +76,6 @@ private String prepareFilterStringForInOperator( } // STRING or null fieldType: no casting needed - return String.format("%s = ANY(ARRAY[%s])", lhsWithCast, placeholders); + return String.format("%s = ANY(?)", lhsWithCast); } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java index 24f37308..0e7ddb51 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java @@ -1,12 +1,12 @@ package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.nonjson.field; -import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; import org.hypertrace.core.documentstore.postgres.Params; import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresInRelationalFilterParserInterface; import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresRelationalFilterParser; +import org.hypertrace.core.documentstore.postgres.utils.PostgresUtils; /** * Implementation of PostgresInRelationalFilterParserInterface for handling IN operations on array @@ -49,29 +49,26 @@ public String parse( /** * Generates SQL for scalar IN operator (used when array field has been unnested). Example: - * "tags_unnested" = ANY(ARRAY[?, ?, ?]) + * "tags_unnested" = ANY(?) */ private String prepareFilterStringForScalarInOperator( final String parsedLhs, final Iterable parsedRhs, final Params.Builder paramsBuilder) { - String placeholders = - StreamSupport.stream(parsedRhs.spliterator(), false) - .map( - value -> { - paramsBuilder.addObjectParam(value); - return "?"; - }) - .collect(Collectors.joining(", ")); - - // Optimized scalar IN operator for unnested array elements - return String.format("%s = ANY(ARRAY[%s])", parsedLhs, placeholders); + Object[] values = StreamSupport.stream(parsedRhs.spliterator(), false).toArray(); + + // SQL type is needed during parameter binding + paramsBuilder.addArrayParam(values, PostgresUtils.inferSqlTypeFromValue(values)); + + return String.format("%s = ANY(?)", parsedLhs); } /** * Generates SQL for array overlap operator (used for non-unnested array fields). Example: "tags" - * && ARRAY[?, ?]::text[] + * && ? + * + *

Uses a single array parameter. */ private String prepareFilterStringForArrayInOperator( final String parsedLhs, @@ -79,31 +76,38 @@ private String prepareFilterStringForArrayInOperator( final String arrayType, final Params.Builder paramsBuilder) { - String placeholders = - StreamSupport.stream(parsedRhs.spliterator(), false) - .map( - value -> { - paramsBuilder.addObjectParam(value); - return "?"; - }) - .collect(Collectors.joining(", ")); - - // Use array overlap operator for array fields - if (arrayType != null) { - // Type-aware optimization - if (arrayType.equals("text[]")) { - // cast RHS to text[] otherwise JDBC binds it as character varying[]. - return String.format("%s && ARRAY[%s]::text[]", parsedLhs, placeholders); - } else { - // INTEGER/BOOLEAN arrays: No casting needed, JDBC binds them correctly - // "numbers" && ARRAY[?, ?] (PostgreSQL infers integer[]) - // "flags" && ARRAY[?, ?] (PostgreSQL infers boolean[]) - return String.format("%s && ARRAY[%s]", parsedLhs, placeholders); - } - } else { - // Fallback: Cast both LHS and RHS to text[] to avoid type mismatch issues. This has the worst - // performance because casting LHS doesn't let PG use indexes on this col - return String.format("%s::text[] && ARRAY[%s]::text[]", parsedLhs, placeholders); + // Collect all values into an array + Object[] values = StreamSupport.stream(parsedRhs.spliterator(), false).toArray(); + + // Infer SQL type from first value or array type hint + String sqlType = + arrayType != null + ? mapArrayTypeToSqlType(arrayType) + : PostgresUtils.inferSqlTypeFromValue(values); + + // Add as single array parameter + paramsBuilder.addArrayParam(values, sqlType); + + // Use array overlap operator with single parameter + return String.format("%s && ?", parsedLhs); + } + + private String mapArrayTypeToSqlType(String arrayType) { + // Remove [] suffix + String baseType = arrayType.replace("[]", ""); + + // Map to internal type names for createArrayOf() + switch (baseType) { + case "double precision": + return "float8"; + case "integer": + return "int4"; + case "boolean": + return "bool"; + case "text": + return "text"; + default: + return baseType; } } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java index bce4e73e..e2fdb0a2 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java @@ -1,11 +1,11 @@ package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.nonjson.field; -import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; import org.hypertrace.core.documentstore.postgres.Params; import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresInRelationalFilterParserInterface; import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresRelationalFilterParser; +import org.hypertrace.core.documentstore.postgres.utils.PostgresUtils; /** * Implementation of PostgresInRelationalFilterParserInterface for handling IN operations on @@ -29,15 +29,10 @@ private String prepareFilterStringForInOperator( final Iterable parsedRhs, final Params.Builder paramsBuilder) { - String placeholders = - StreamSupport.stream(parsedRhs.spliterator(), false) - .map( - value -> { - paramsBuilder.addObjectParam(value); - return "?"; - }) - .collect(Collectors.joining(", ")); + Object[] values = StreamSupport.stream(parsedRhs.spliterator(), false).toArray(); - return String.format("%s = ANY(ARRAY[%s])", parsedLhs, placeholders); + paramsBuilder.addArrayParam(values, PostgresUtils.inferSqlTypeFromValue(values)); + + return String.format("%s = ANY(?)", parsedLhs); } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java index 24b8ffc6..f8212fd0 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java @@ -14,6 +14,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; +import java.sql.Array; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.Arrays; @@ -591,6 +592,30 @@ public static DocumentAndId extractAndRemoveId(final Document document) throws I return new DocumentAndId(documentWithoutId, id); } + /** + * Infers PostgreSQL SQL type name for createArrayOf() from Java array values. Inspects the first + * element to determine the appropriate type. + * + * @param values Array of values to infer type from + * @return PostgreSQL internal type name: "int4", "float8", "bool", or "text" + */ + public static String inferSqlTypeFromValue(Object[] values) { + if (values.length == 0) { + return "text"; + } + + Object firstValue = values[0]; + if (firstValue instanceof Integer || firstValue instanceof Long) { + return "int4"; + } else if (firstValue instanceof Double || firstValue instanceof Float) { + return "float8"; + } else if (firstValue instanceof Boolean) { + return "bool"; + } else { + return "text"; + } + } + public static void enrichPreparedStatementWithParams( final PreparedStatement preparedStatement, final Params params) { params @@ -598,13 +623,22 @@ public static void enrichPreparedStatementWithParams( .forEach( (k, v) -> { try { - if (isValidPrimitiveType(v)) { + if (v instanceof Params.ArrayParam) { + // Handle array parameter - create PostgreSQL array + Params.ArrayParam arrayParam = (Params.ArrayParam) v; + Array sqlArray = + preparedStatement + .getConnection() + .createArrayOf(arrayParam.getSqlType(), arrayParam.getValues()); + preparedStatement.setArray(k, sqlArray); + } else if (isValidPrimitiveType(v)) { preparedStatement.setObject(k, v); } else { throw new UnsupportedOperationException("Un-supported object types in filter"); } } catch (SQLException e) { log.error("SQLException setting Param. key: {}, value: {}", k, v); + throw new RuntimeException("Failed to set parameter " + k, e); } }); if (log.isDebugEnabled()) { diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java index f698de3a..8d798177 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java @@ -684,7 +684,7 @@ void testFindWithSortingAndPagination() { + "\"quantity\" AS \"quantity\", " + "\"date\" AS \"date\" " + "FROM \"testCollection\" " - + "WHERE \"item\" = ANY(ARRAY[?, ?, ?, ?]) " + + "WHERE \"item\" = ANY(?) " + "ORDER BY \"quantity\" DESC NULLS LAST,\"item\" ASC NULLS FIRST " + "OFFSET ? LIMIT ?", postgresQueryParser.parse()); @@ -1506,11 +1506,11 @@ void testNotInWithFlatCollectionNonJsonField() { String sql = postgresQueryParser.parse(); assertEquals( - "SELECT * FROM \"testCollection\" WHERE \"category\" IS NULL OR NOT (\"category\" = ANY(ARRAY[?, ?]))", + "SELECT * FROM \"testCollection\" WHERE \"category\" IS NULL OR NOT (\"category\" = ANY(?))", sql); Params params = postgresQueryParser.getParamsBuilder().build(); - assertEquals(2, params.getObjectParams().size()); + assertEquals(1, params.getObjectParams().size()); } @Test @@ -1809,4 +1809,141 @@ void testFlatCollectionWithHyphenatedJsonbArrayFieldInUnnest() { Params params = postgresQueryParser.getParamsBuilder().build(); assertEquals("team-alpha", params.getObjectParams().get(1)); } + + @Test + void testInOperatorWithScalarStringField() { + Query query = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("category"), + IN, + ConstantExpression.ofStrings(List.of("electronics", "clothing", "books")))) + .build(); + + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser( + TEST_TABLE, + PostgresQueryTransformer.transform(query), + new FlatPostgresFieldTransformer()); + + String sql = postgresQueryParser.parse(); + assertEquals("SELECT * FROM \"testCollection\" WHERE \"category\" = ANY(?)", sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + assertEquals(1, params.getObjectParams().size()); + Params.ArrayParam arrayParam = (Params.ArrayParam) params.getObjectParams().get(1); + assertEquals("text", arrayParam.getSqlType()); + assertEquals(3, arrayParam.getValues().length); + } + + @Test + void testInOperatorWithScalarIntegerField() { + Query query = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("price"), + IN, + ConstantExpression.ofNumbers(List.of(10, 20, 30)))) + .build(); + + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser( + TEST_TABLE, + PostgresQueryTransformer.transform(query), + new FlatPostgresFieldTransformer()); + + String sql = postgresQueryParser.parse(); + assertEquals("SELECT * FROM \"testCollection\" WHERE \"price\" = ANY(?)", sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + assertEquals(1, params.getObjectParams().size()); + Params.ArrayParam arrayParam = (Params.ArrayParam) params.getObjectParams().get(1); + assertEquals("int4", arrayParam.getSqlType()); + assertEquals(3, arrayParam.getValues().length); + } + + @Test + void testInOperatorWithTextArrayField() { + Query query = + Query.builder() + .setFilter( + RelationalExpression.of( + ArrayIdentifierExpression.of("tags", ArrayType.TEXT), + IN, + ConstantExpression.ofStrings(List.of("premium", "sale", "new")))) + .build(); + + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser( + TEST_TABLE, + PostgresQueryTransformer.transform(query), + new FlatPostgresFieldTransformer()); + + String sql = postgresQueryParser.parse(); + assertEquals("SELECT * FROM \"testCollection\" WHERE \"tags\" && ?", sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + assertEquals(1, params.getObjectParams().size()); + Params.ArrayParam arrayParam = (Params.ArrayParam) params.getObjectParams().get(1); + assertEquals("text", arrayParam.getSqlType()); + assertEquals(3, arrayParam.getValues().length); + } + + @Test + void testInOperatorWithIntegerArrayField() { + Query query = + Query.builder() + .setFilter( + RelationalExpression.of( + ArrayIdentifierExpression.of("numbers", ArrayType.INTEGER), + IN, + ConstantExpression.ofNumbers(List.of(5, 10, 15)))) + .build(); + + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser( + TEST_TABLE, + PostgresQueryTransformer.transform(query), + new FlatPostgresFieldTransformer()); + + String sql = postgresQueryParser.parse(); + assertEquals("SELECT * FROM \"testCollection\" WHERE \"numbers\" && ?", sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + assertEquals(1, params.getObjectParams().size()); + Params.ArrayParam arrayParam = (Params.ArrayParam) params.getObjectParams().get(1); + assertEquals("int4", arrayParam.getSqlType()); + assertEquals(3, arrayParam.getValues().length); + } + + @Test + void testNotInOperatorWithScalarStringField() { + Query query = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("status"), + NOT_IN, + ConstantExpression.ofStrings(List.of("inactive", "archived")))) + .build(); + + PostgresQueryParser postgresQueryParser = + new PostgresQueryParser( + TEST_TABLE, + PostgresQueryTransformer.transform(query), + new FlatPostgresFieldTransformer()); + + String sql = postgresQueryParser.parse(); + assertEquals( + "SELECT * FROM \"testCollection\" WHERE \"status\" IS NULL OR NOT (\"status\" = ANY(?))", + sql); + + Params params = postgresQueryParser.getParamsBuilder().build(); + assertEquals(1, params.getObjectParams().size()); + Params.ArrayParam arrayParam = (Params.ArrayParam) params.getObjectParams().get(1); + assertEquals("text", arrayParam.getSqlType()); + assertEquals(2, arrayParam.getValues().length); + } } From 6b8579a548285db398078587453ac855732aabb9 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Thu, 11 Dec 2025 15:27:22 +0530 Subject: [PATCH 4/9] WIP --- .../core/documentstore/postgres/utils/PostgresUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java index f8212fd0..b2e502d7 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java @@ -638,7 +638,6 @@ public static void enrichPreparedStatementWithParams( } } catch (SQLException e) { log.error("SQLException setting Param. key: {}, value: {}", k, v); - throw new RuntimeException("Failed to set parameter " + k, e); } }); if (log.isDebugEnabled()) { From 201b442af4507a6a68877778bb947a5118a8143c Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Thu, 11 Dec 2025 17:10:12 +0530 Subject: [PATCH 5/9] Handle empty arrays --- .../PostgresInRelationalFilterParserJsonPrimitive.java | 7 +++++-- .../PostgresInRelationalFilterParserArrayField.java | 9 +++++++++ .../PostgresInRelationalFilterParserScalarField.java | 5 +++++ .../core/documentstore/postgres/utils/PostgresUtils.java | 4 ++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java index 81835c67..150b3875 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java @@ -64,6 +64,11 @@ private String prepareFilterStringForInOperator( Object[] values = StreamSupport.stream(parsedRhs.spliterator(), false).toArray(); + if (values.length == 0) { + // return FALSE + return "1 = 0"; + } + // Add as single array parameter paramsBuilder.addArrayParam(values, PostgresUtils.inferSqlTypeFromValue(values)); @@ -74,8 +79,6 @@ private String prepareFilterStringForInOperator( } else if (fieldType == JsonFieldType.BOOLEAN) { lhsWithCast = String.format("CAST(%s AS BOOLEAN)", parsedLhs); } - // STRING or null fieldType: no casting needed - return String.format("%s = ANY(?)", lhsWithCast); } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java index 0e7ddb51..3da5f964 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java @@ -58,6 +58,11 @@ private String prepareFilterStringForScalarInOperator( Object[] values = StreamSupport.stream(parsedRhs.spliterator(), false).toArray(); + if (values.length == 0) { + // return FALSE + return "1 = 0"; + } + // SQL type is needed during parameter binding paramsBuilder.addArrayParam(values, PostgresUtils.inferSqlTypeFromValue(values)); @@ -79,6 +84,10 @@ private String prepareFilterStringForArrayInOperator( // Collect all values into an array Object[] values = StreamSupport.stream(parsedRhs.spliterator(), false).toArray(); + if (values.length == 0) { + return "1 = 0"; + } + // Infer SQL type from first value or array type hint String sqlType = arrayType != null diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java index e2fdb0a2..d7b0964e 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java @@ -31,6 +31,11 @@ private String prepareFilterStringForInOperator( Object[] values = StreamSupport.stream(parsedRhs.spliterator(), false).toArray(); + if (values.length == 0) { + // return FALSE + return "1 = 0"; + } + paramsBuilder.addArrayParam(values, PostgresUtils.inferSqlTypeFromValue(values)); return String.format("%s = ANY(?)", parsedLhs); diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java index b2e502d7..e3b1482f 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java @@ -596,6 +596,10 @@ public static DocumentAndId extractAndRemoveId(final Document document) throws I * Infers PostgreSQL SQL type name for createArrayOf() from Java array values. Inspects the first * element to determine the appropriate type. * + *

Note: Empty arrays should be handled by the caller before SQL generation to avoid + * PostgreSQL type mismatch errors (e.g., {@code integer = text[]}). Filter parsers should return + * {@code "1 = 0"} for empty IN clauses instead of calling this method. + * * @param values Array of values to infer type from * @return PostgreSQL internal type name: "int4", "float8", "bool", or "text" */ From 211fb41c527d2cdbe0162c335bfebd4961591af0 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Thu, 18 Dec 2025 16:39:47 +0530 Subject: [PATCH 6/9] Remove instanceof checks by using compile-type type --- ...InRelationalFilterParserJsonPrimitive.java | 17 ++- ...gresTopLevelArrayEqualityFilterParser.java | 4 +- ...insRelationalFilterParserNonJsonField.java | 15 +-- .../nonjson/field/PostgresDataType.java | 23 ++-- ...resInRelationalFilterParserArrayField.java | 108 ++++++++++-------- ...esInRelationalFilterParserScalarField.java | 48 +++++++- ...ractor.java => PostgresTypeExtractor.java} | 66 ++++++++--- .../postgres/utils/PostgresUtils.java | 28 ----- .../impl/ArrayIdentifierExpressionTest.java | 32 +++--- .../query/v1/PostgresQueryParserTest.java | 14 +-- ...ostgresPostgresArrayTypeExtractorTest.java | 100 ---------------- 11 files changed, 205 insertions(+), 250 deletions(-) rename document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/{PostgresArrayTypeExtractor.java => PostgresTypeExtractor.java} (55%) delete mode 100644 document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresPostgresArrayTypeExtractorTest.java diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java index 150b3875..915a513b 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresInRelationalFilterParserJsonPrimitive.java @@ -5,7 +5,6 @@ import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; import org.hypertrace.core.documentstore.postgres.Params; -import org.hypertrace.core.documentstore.postgres.utils.PostgresUtils; /** * Optimized parser for IN operations on JSON primitive fields (string, number, boolean) with proper @@ -69,8 +68,8 @@ private String prepareFilterStringForInOperator( return "1 = 0"; } - // Add as single array parameter - paramsBuilder.addArrayParam(values, PostgresUtils.inferSqlTypeFromValue(values)); + String sqlType = mapJsonFieldTypeToSqlType(fieldType); + paramsBuilder.addArrayParam(values, sqlType); // Apply appropriate casting based on field type String lhsWithCast = parsedLhs; @@ -81,4 +80,16 @@ private String prepareFilterStringForInOperator( } return String.format("%s = ANY(?)", lhsWithCast); } + + private String mapJsonFieldTypeToSqlType(JsonFieldType fieldType) { + switch (fieldType) { + case NUMBER: + return "float8"; + case BOOLEAN: + return "bool"; + case STRING: + default: + return "text"; + } + } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresTopLevelArrayEqualityFilterParser.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresTopLevelArrayEqualityFilterParser.java index 7fd7ead6..302e6feb 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresTopLevelArrayEqualityFilterParser.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/PostgresTopLevelArrayEqualityFilterParser.java @@ -5,7 +5,7 @@ import java.util.stream.StreamSupport; import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; -import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.nonjson.field.PostgresArrayTypeExtractor; +import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.nonjson.field.PostgresTypeExtractor; /** * Handles EQ/NEQ operations on top-level array columns when RHS is also an array, using exact @@ -43,7 +43,7 @@ public String parse( .collect(Collectors.joining(", ")); ArrayIdentifierExpression arrayExpr = (ArrayIdentifierExpression) expression.getLhs(); - String arrayTypeCast = arrayExpr.accept(new PostgresArrayTypeExtractor()); + String arrayTypeCast = arrayExpr.accept(PostgresTypeExtractor.arrayType()); // Generate: tags = ARRAY[?, ?]::text[] if (arrayTypeCast != null) { diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresContainsRelationalFilterParserNonJsonField.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresContainsRelationalFilterParserNonJsonField.java index 1db11c49..d00e2862 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresContainsRelationalFilterParserNonJsonField.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresContainsRelationalFilterParserNonJsonField.java @@ -50,18 +50,11 @@ public String parse( } // Field is NOT unnested - use array containment operator - String arrayTypeCast = expression.getLhs().accept(new PostgresArrayTypeExtractor()); + String arrayType = expression.getLhs().accept(PostgresTypeExtractor.arrayType()); + // Fallback to text[] if type is unknown + String typeCast = (arrayType != null) ? arrayType : "text[]"; - // Use ARRAY[?, ?, ...] syntax with appropriate type cast - if (arrayTypeCast != null && arrayTypeCast.equals("text[]")) { - return String.format("%s @> ARRAY[%s]::text[]", parsedLhs, placeholders); - } else if (arrayTypeCast != null) { - // INTEGER/BOOLEAN/DOUBLE arrays: Use the correct type cast - return String.format("%s @> ARRAY[%s]::%s", parsedLhs, placeholders, arrayTypeCast); - } else { - // Fallback: use text[] cast - return String.format("%s @> ARRAY[%s]::text[]", parsedLhs, placeholders); - } + return String.format("%s @> ARRAY[%s]::%s", parsedLhs, placeholders, typeCast); } private Iterable normalizeToIterable(final Object value) { diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresDataType.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresDataType.java index d59a35ee..89626e7c 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresDataType.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresDataType.java @@ -5,19 +5,19 @@ /** * PostgreSQL-specific data types with their SQL type strings. * - *

This enum maps generic {@link DataType} values to PostgreSQL-specific type strings used in SQL - * queries for type casting. + *

This enum maps generic {@link DataType} values to PostgreSQL internal type names, which work + * for both JDBC's {@code Connection.createArrayOf()} and SQL type casting. */ public enum PostgresDataType { TEXT("text"), - INTEGER("integer"), - BIGINT("bigint"), - REAL("real"), - DOUBLE_PRECISION("double precision"), - BOOLEAN("boolean"), + INTEGER("int4"), + BIGINT("int8"), + REAL("float4"), + DOUBLE_PRECISION("float8"), + BOOLEAN("bool"), TIMESTAMPTZ("timestamptz"), DATE("date"), - UNKNOWN("unknown"); + UNKNOWN(null); private final String sqlType; @@ -25,10 +25,16 @@ public enum PostgresDataType { this.sqlType = sqlType; } + /** + * Returns the PostgreSQL type name for use with JDBC's createArrayOf() and SQL casting. + * + * @return The type name (e.g., "int4", "float8", "text") + */ public String getSqlType() { return sqlType; } + /** Returns the array type for SQL casting (e.g., "int4[]", "text[]"). */ public String getArraySqlType() { return sqlType + "[]"; } @@ -38,7 +44,6 @@ public String getArraySqlType() { * * @param dataType the generic data type * @return the corresponding PostgresDataType, or null if UNSPECIFIED - * @throws IllegalArgumentException if the DataType is unknown */ public static PostgresDataType fromDataType(DataType dataType) { switch (dataType) { diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java index 3da5f964..20b2f093 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java @@ -1,12 +1,12 @@ package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.nonjson.field; +import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; import org.hypertrace.core.documentstore.postgres.Params; import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresInRelationalFilterParserInterface; import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresRelationalFilterParser; -import org.hypertrace.core.documentstore.postgres.utils.PostgresUtils; /** * Implementation of PostgresInRelationalFilterParserInterface for handling IN operations on array @@ -31,6 +31,9 @@ public String parse( final String parsedLhs = expression.getLhs().accept(context.lhsParser()); final Iterable parsedRhs = expression.getRhs().accept(context.rhsParser()); + // Extract element type from expression metadata for type-safe query generation + String sqlType = expression.getLhs().accept(PostgresTypeExtractor.scalarType()); + // Check if this field has been unnested - if so, treat it as a scalar ArrayIdentifierExpression arrayExpr = (ArrayIdentifierExpression) expression.getLhs(); String fieldName = arrayExpr.getName(); @@ -38,13 +41,12 @@ public String parse( // Field is unnested - each element is now a scalar, not an array // Use scalar IN operator instead of array overlap return prepareFilterStringForScalarInOperator( - parsedLhs, parsedRhs, context.getParamsBuilder()); + parsedLhs, parsedRhs, sqlType, context.getParamsBuilder()); } // Field is NOT unnested - use array overlap logic - String arrayTypeCast = expression.getLhs().accept(new PostgresArrayTypeExtractor()); return prepareFilterStringForArrayInOperator( - parsedLhs, parsedRhs, arrayTypeCast, context.getParamsBuilder()); + parsedLhs, parsedRhs, sqlType, context.getParamsBuilder()); } /** @@ -54,19 +56,22 @@ public String parse( private String prepareFilterStringForScalarInOperator( final String parsedLhs, final Iterable parsedRhs, + final String sqlType, final Params.Builder paramsBuilder) { - - Object[] values = StreamSupport.stream(parsedRhs.spliterator(), false).toArray(); - - if (values.length == 0) { - // return FALSE - return "1 = 0"; + // If type is specified, use optimized ANY(ARRAY[]) syntax + // Otherwise, fall back to traditional IN (?, ?, ?) for backward compatibility + if (sqlType != null) { + Object[] values = StreamSupport.stream(parsedRhs.spliterator(), false).toArray(); + + if (values.length == 0) { + return "1 = 0"; + } + + paramsBuilder.addArrayParam(values, sqlType); + return String.format("%s = ANY(?)", parsedLhs); + } else { + return prepareFilterStringFallback(parsedLhs, parsedRhs, paramsBuilder, "%s IN (%s)"); } - - // SQL type is needed during parameter binding - paramsBuilder.addArrayParam(values, PostgresUtils.inferSqlTypeFromValue(values)); - - return String.format("%s = ANY(?)", parsedLhs); } /** @@ -78,45 +83,48 @@ private String prepareFilterStringForScalarInOperator( private String prepareFilterStringForArrayInOperator( final String parsedLhs, final Iterable parsedRhs, - final String arrayType, + final String sqlType, final Params.Builder paramsBuilder) { - - // Collect all values into an array - Object[] values = StreamSupport.stream(parsedRhs.spliterator(), false).toArray(); - - if (values.length == 0) { - return "1 = 0"; + // If type is specified, use optimized array overlap with typed array + // Otherwise, fall back to jsonb-based approach for backward compatibility + if (sqlType != null) { + Object[] values = StreamSupport.stream(parsedRhs.spliterator(), false).toArray(); + + if (values.length == 0) { + return "1 = 0"; + } + + paramsBuilder.addArrayParam(values, sqlType); + return String.format("%s && ?", parsedLhs); + } else { + // Fallback: use jsonb array contains approach + return prepareFilterStringFallback(parsedLhs, parsedRhs, paramsBuilder, "%s @> ARRAY[%s]"); } - - // Infer SQL type from first value or array type hint - String sqlType = - arrayType != null - ? mapArrayTypeToSqlType(arrayType) - : PostgresUtils.inferSqlTypeFromValue(values); - - // Add as single array parameter - paramsBuilder.addArrayParam(values, sqlType); - - // Use array overlap operator with single parameter - return String.format("%s && ?", parsedLhs); } - private String mapArrayTypeToSqlType(String arrayType) { - // Remove [] suffix - String baseType = arrayType.replace("[]", ""); - - // Map to internal type names for createArrayOf() - switch (baseType) { - case "double precision": - return "float8"; - case "integer": - return "int4"; - case "boolean": - return "bool"; - case "text": - return "text"; - default: - return baseType; + /** + * Fallback method using traditional (?, ?, ?) syntax for backward compatibility when type + * information is not available. + */ + private String prepareFilterStringFallback( + final String parsedLhs, + final Iterable parsedRhs, + final Params.Builder paramsBuilder, + final String formatPattern) { + + String collect = + StreamSupport.stream(parsedRhs.spliterator(), false) + .map( + val -> { + paramsBuilder.addObjectParam(val); + return "?"; + }) + .collect(Collectors.joining(", ")); + + if (collect.isEmpty()) { + return "1 = 0"; } + + return String.format(formatPattern, parsedLhs, collect); } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java index d7b0964e..55108aa2 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserScalarField.java @@ -1,11 +1,11 @@ package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.nonjson.field; +import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.hypertrace.core.documentstore.expression.impl.RelationalExpression; import org.hypertrace.core.documentstore.postgres.Params; import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresInRelationalFilterParserInterface; import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.PostgresRelationalFilterParser; -import org.hypertrace.core.documentstore.postgres.utils.PostgresUtils; /** * Implementation of PostgresInRelationalFilterParserInterface for handling IN operations on @@ -21,23 +21,61 @@ public String parse( final String parsedLhs = expression.getLhs().accept(context.lhsParser()); final Iterable parsedRhs = expression.getRhs().accept(context.rhsParser()); - return prepareFilterStringForInOperator(parsedLhs, parsedRhs, context.getParamsBuilder()); + // Extract type from expression metadata + String sqlType = expression.getLhs().accept(PostgresTypeExtractor.scalarType()); + + // If type is specified, use optimized ANY(ARRAY[]) syntax + // Otherwise, fall back to traditional IN (?, ?, ?) for backward compatibility + if (sqlType != null) { + return prepareFilterStringForInOperatorWithArray( + parsedLhs, parsedRhs, sqlType, context.getParamsBuilder()); + } else { + return prepareFilterStringForInOperatorFallback( + parsedLhs, parsedRhs, context.getParamsBuilder()); + } } - private String prepareFilterStringForInOperator( + /** Optimized IN using = ANY(?) with typed array parameter. */ + private String prepareFilterStringForInOperatorWithArray( final String parsedLhs, final Iterable parsedRhs, + final String sqlType, final Params.Builder paramsBuilder) { Object[] values = StreamSupport.stream(parsedRhs.spliterator(), false).toArray(); if (values.length == 0) { - // return FALSE + // Evaluates to FALSE return "1 = 0"; } - paramsBuilder.addArrayParam(values, PostgresUtils.inferSqlTypeFromValue(values)); + paramsBuilder.addArrayParam(values, sqlType); return String.format("%s = ANY(?)", parsedLhs); } + + /** + * Fallback IN using traditional IN (?, ?, ?) syntax for backward compatibility when type + * information is not available. + */ + private String prepareFilterStringForInOperatorFallback( + final String parsedLhs, + final Iterable parsedRhs, + final Params.Builder paramsBuilder) { + + String collect = + StreamSupport.stream(parsedRhs.spliterator(), false) + .map( + val -> { + paramsBuilder.addObjectParam(val); + return "?"; + }) + .collect(Collectors.joining(", ")); + + if (collect.isEmpty()) { + return "1 = 0"; + } + + return String.format("%s IN (%s)", parsedLhs, collect); + } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresArrayTypeExtractor.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresTypeExtractor.java similarity index 55% rename from document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresArrayTypeExtractor.java rename to document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresTypeExtractor.java index 1e6c5160..9a163255 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresArrayTypeExtractor.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresTypeExtractor.java @@ -11,27 +11,61 @@ import org.hypertrace.core.documentstore.parser.SelectTypeExpressionVisitor; /** - * Visitor to extract PostgreSQL array type information from {@link ArrayIdentifierExpression}. + * Visitor to extract PostgreSQL type information from identifier expressions. * - *

This visitor is specifically designed to work ONLY with {@link ArrayIdentifierExpression}. Any - * other expression type will throw {@link UnsupportedOperationException} to catch programming - * errors early. - * - *

Returns: + *

Supports two extraction modes: * *

    - *
  • The PostgreSQL array type string (e.g., "text[]", "integer[]") - *
  • {@code null} if {@link ArrayIdentifierExpression} has UNSPECIFIED type + *
  • Scalar type: Returns type names (e.g., "int4", "text") for use with {@code + * Connection.createArrayOf()} + *
  • Array type: Returns array type strings (e.g., "int4[]", "text[]") for SQL type + * casting *
+ * + *

Returns {@code null} if the expression has UNSPECIFIED type. */ -public class PostgresArrayTypeExtractor implements SelectTypeExpressionVisitor { +public class PostgresTypeExtractor implements SelectTypeExpressionVisitor { + + private final boolean extractArrayType; + + private PostgresTypeExtractor(boolean extractArrayType) { + this.extractArrayType = extractArrayType; + } + + /** + * Creates an extractor that returns scalar SQL type names. + * + * @return A type extractor returning types like "int4", "float8", "text" + */ + public static PostgresTypeExtractor scalarType() { + return new PostgresTypeExtractor(false); + } + + /** + * Creates an extractor that returns array SQL type strings for SQL type casting. + * + * @return A type extractor returning array types like "int4[]", "text[]" + */ + public static PostgresTypeExtractor arrayType() { + return new PostgresTypeExtractor(true); + } - public PostgresArrayTypeExtractor() {} + @Override + public String visit(IdentifierExpression expression) { + PostgresDataType pgType = PostgresDataType.fromDataType(expression.getDataType()); + if (pgType == PostgresDataType.UNKNOWN) { + return null; + } + return extractArrayType ? pgType.getArraySqlType() : pgType.getSqlType(); + } @Override public String visit(ArrayIdentifierExpression expression) { PostgresDataType pgType = PostgresDataType.fromDataType(expression.getElementDataType()); - return pgType == PostgresDataType.UNKNOWN ? null : pgType.getArraySqlType(); + if (pgType == PostgresDataType.UNKNOWN) { + return null; + } + return extractArrayType ? pgType.getArraySqlType() : pgType.getSqlType(); } @Override @@ -39,13 +73,6 @@ public String visit(JsonIdentifierExpression expression) { throw unsupportedExpression("JsonIdentifierExpression"); } - @Override - public String visit(IdentifierExpression expression) { - throw new UnsupportedOperationException( - "PostgresArrayTypeExtractor should only be used with ArrayIdentifierExpression. " - + "Use IdentifierExpression only for scalar fields, not arrays."); - } - @Override public String visit(AggregateExpression expression) { throw unsupportedExpression("AggregateExpression"); @@ -73,7 +100,8 @@ public String visit(AliasedIdentifierExpression expression) { private static UnsupportedOperationException unsupportedExpression(String expressionType) { return new UnsupportedOperationException( - "PostgresArrayTypeExtractor should only be used with ArrayIdentifierExpression, not " + "PostgresTypeExtractor should only be used with IdentifierExpression or " + + "ArrayIdentifierExpression, not " + expressionType); } } diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java index e3b1482f..5e8e64cf 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/utils/PostgresUtils.java @@ -592,34 +592,6 @@ public static DocumentAndId extractAndRemoveId(final Document document) throws I return new DocumentAndId(documentWithoutId, id); } - /** - * Infers PostgreSQL SQL type name for createArrayOf() from Java array values. Inspects the first - * element to determine the appropriate type. - * - *

Note: Empty arrays should be handled by the caller before SQL generation to avoid - * PostgreSQL type mismatch errors (e.g., {@code integer = text[]}). Filter parsers should return - * {@code "1 = 0"} for empty IN clauses instead of calling this method. - * - * @param values Array of values to infer type from - * @return PostgreSQL internal type name: "int4", "float8", "bool", or "text" - */ - public static String inferSqlTypeFromValue(Object[] values) { - if (values.length == 0) { - return "text"; - } - - Object firstValue = values[0]; - if (firstValue instanceof Integer || firstValue instanceof Long) { - return "int4"; - } else if (firstValue instanceof Double || firstValue instanceof Float) { - return "float8"; - } else if (firstValue instanceof Boolean) { - return "bool"; - } else { - return "text"; - } - } - public static void enrichPreparedStatementWithParams( final PreparedStatement preparedStatement, final Params params) { params diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/ArrayIdentifierExpressionTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/ArrayIdentifierExpressionTest.java index 2e0e7375..f80c0aee 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/ArrayIdentifierExpressionTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/expression/impl/ArrayIdentifierExpressionTest.java @@ -4,7 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNull; -import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.nonjson.field.PostgresArrayTypeExtractor; +import org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.nonjson.field.PostgresTypeExtractor; import org.junit.jupiter.api.Test; class ArrayIdentifierExpressionTest { @@ -79,7 +79,7 @@ void testOfStringsCreatesInstanceWithTextType() { assertEquals("tags", expression.getName()); assertEquals(DataType.STRING, expression.getElementDataType()); - PostgresArrayTypeExtractor extractor = new PostgresArrayTypeExtractor(); + PostgresTypeExtractor extractor = PostgresTypeExtractor.arrayType(); assertEquals("text[]", expression.accept(extractor)); } @@ -90,8 +90,8 @@ void testOfIntsCreatesInstanceWithIntegerType() { assertEquals("ids", expression.getName()); assertEquals(DataType.INTEGER, expression.getElementDataType()); - PostgresArrayTypeExtractor extractor = new PostgresArrayTypeExtractor(); - assertEquals("integer[]", expression.accept(extractor)); + PostgresTypeExtractor extractor = PostgresTypeExtractor.arrayType(); + assertEquals("int4[]", expression.accept(extractor)); } @Test @@ -101,8 +101,8 @@ void testOfLongsCreatesInstanceWithBigintType() { assertEquals("timestamps", expression.getName()); assertEquals(DataType.LONG, expression.getElementDataType()); - PostgresArrayTypeExtractor extractor = new PostgresArrayTypeExtractor(); - assertEquals("bigint[]", expression.accept(extractor)); + PostgresTypeExtractor extractor = PostgresTypeExtractor.arrayType(); + assertEquals("int8[]", expression.accept(extractor)); } @Test @@ -112,8 +112,8 @@ void testOfFloatsCreatesInstanceWithFloatType() { assertEquals("temperatures", expression.getName()); assertEquals(DataType.FLOAT, expression.getElementDataType()); - PostgresArrayTypeExtractor extractor = new PostgresArrayTypeExtractor(); - assertEquals("real[]", expression.accept(extractor)); + PostgresTypeExtractor extractor = PostgresTypeExtractor.arrayType(); + assertEquals("float4[]", expression.accept(extractor)); } @Test @@ -123,8 +123,8 @@ void testOfDoublesCreatesInstanceWithDoubleType() { assertEquals("coordinates", expression.getName()); assertEquals(DataType.DOUBLE, expression.getElementDataType()); - PostgresArrayTypeExtractor extractor = new PostgresArrayTypeExtractor(); - assertEquals("double precision[]", expression.accept(extractor)); + PostgresTypeExtractor extractor = PostgresTypeExtractor.arrayType(); + assertEquals("float8[]", expression.accept(extractor)); } @Test @@ -134,8 +134,8 @@ void testOfBooleansCreatesInstanceWithBooleanType() { assertEquals("flags", expression.getName()); assertEquals(DataType.BOOLEAN, expression.getElementDataType()); - PostgresArrayTypeExtractor extractor = new PostgresArrayTypeExtractor(); - assertEquals("boolean[]", expression.accept(extractor)); + PostgresTypeExtractor extractor = PostgresTypeExtractor.arrayType(); + assertEquals("bool[]", expression.accept(extractor)); } @Test @@ -145,7 +145,7 @@ void testOfTimestampsTzCreatesInstanceWithTimestampTzType() { assertEquals("eventsTz", expression.getName()); assertEquals(DataType.TIMESTAMPTZ, expression.getElementDataType()); - PostgresArrayTypeExtractor extractor = new PostgresArrayTypeExtractor(); + PostgresTypeExtractor extractor = PostgresTypeExtractor.arrayType(); assertEquals("timestamptz[]", expression.accept(extractor)); } @@ -156,17 +156,17 @@ void testOfDatesCreatesInstanceWithDateType() { assertEquals("dates", expression.getName()); assertEquals(DataType.DATE, expression.getElementDataType()); - PostgresArrayTypeExtractor extractor = new PostgresArrayTypeExtractor(); + PostgresTypeExtractor extractor = PostgresTypeExtractor.arrayType(); assertEquals("date[]", expression.accept(extractor)); } @Test - void testPostgresArrayTypeExtractorReturnsNullForUnspecifiedType() { + void testPostgresTypeExtractorReturnsNullForUnspecifiedType() { ArrayIdentifierExpression expression = ArrayIdentifierExpression.of("untyped"); assertEquals(DataType.UNSPECIFIED, expression.getElementDataType()); - PostgresArrayTypeExtractor extractor = new PostgresArrayTypeExtractor(); + PostgresTypeExtractor extractor = PostgresTypeExtractor.arrayType(); String arrayType = expression.accept(extractor); assertNull(arrayType); } diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java index 57aa2d19..74999e39 100644 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/PostgresQueryParserTest.java @@ -625,7 +625,7 @@ void testFindWithSortingAndPagination() { Filter.builder() .expression( RelationalExpression.of( - IdentifierExpression.of("item"), + IdentifierExpression.ofString("item"), IN, ConstantExpression.ofStrings(List.of("Mirror", "Comb", "Shampoo", "Bottle")))) .build(); @@ -1492,7 +1492,7 @@ void testNotInWithFlatCollectionNonJsonField() { Query.builder() .setFilter( RelationalExpression.of( - IdentifierExpression.of("category"), + IdentifierExpression.ofString("category"), NOT_IN, ConstantExpression.ofStrings(List.of("electronics", "clothing")))) .build(); @@ -1815,7 +1815,7 @@ void testInOperatorWithScalarStringField() { Query.builder() .setFilter( RelationalExpression.of( - IdentifierExpression.of("category"), + IdentifierExpression.ofString("category"), IN, ConstantExpression.ofStrings(List.of("electronics", "clothing", "books")))) .build(); @@ -1842,7 +1842,7 @@ void testInOperatorWithScalarIntegerField() { Query.builder() .setFilter( RelationalExpression.of( - IdentifierExpression.of("price"), + IdentifierExpression.ofInt("price"), IN, ConstantExpression.ofNumbers(List.of(10, 20, 30)))) .build(); @@ -1869,7 +1869,7 @@ void testInOperatorWithTextArrayField() { Query.builder() .setFilter( RelationalExpression.of( - ArrayIdentifierExpression.of("tags", ArrayType.TEXT), + ArrayIdentifierExpression.ofStrings("tags"), IN, ConstantExpression.ofStrings(List.of("premium", "sale", "new")))) .build(); @@ -1896,7 +1896,7 @@ void testInOperatorWithIntegerArrayField() { Query.builder() .setFilter( RelationalExpression.of( - ArrayIdentifierExpression.of("numbers", ArrayType.INTEGER), + ArrayIdentifierExpression.ofInts("numbers"), IN, ConstantExpression.ofNumbers(List.of(5, 10, 15)))) .build(); @@ -1923,7 +1923,7 @@ void testNotInOperatorWithScalarStringField() { Query.builder() .setFilter( RelationalExpression.of( - IdentifierExpression.of("status"), + IdentifierExpression.ofString("status"), NOT_IN, ConstantExpression.ofStrings(List.of("inactive", "archived")))) .build(); diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresPostgresArrayTypeExtractorTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresPostgresArrayTypeExtractorTest.java deleted file mode 100644 index d3088838..00000000 --- a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresPostgresArrayTypeExtractorTest.java +++ /dev/null @@ -1,100 +0,0 @@ -package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.nonjson.field; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; -import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; -import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; -import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; -import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; -import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; -import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression; -import org.hypertrace.core.documentstore.expression.operators.FunctionOperator; -import org.junit.jupiter.api.Test; - -class PostgresPostgresDataTypeExtractorTest { - - private final PostgresArrayTypeExtractor extractor = new PostgresArrayTypeExtractor(); - - @Test - void testVisitArrayIdentifierExpression_withType() { - ArrayIdentifierExpression expr = ArrayIdentifierExpression.ofStrings("tags"); - String result = extractor.visit(expr); - assertEquals("text[]", result); - } - - @Test - void testVisitArrayIdentifierExpression_withIntegerType() { - ArrayIdentifierExpression expr = ArrayIdentifierExpression.ofInts("numbers"); - String result = extractor.visit(expr); - assertEquals("integer[]", result); - } - - @Test - void testVisitArrayIdentifierExpression_withBooleanType() { - ArrayIdentifierExpression expr = ArrayIdentifierExpression.ofBooleans("flags"); - String result = extractor.visit(expr); - assertEquals("boolean[]", result); - } - - @Test - void testVisitArrayIdentifierExpression_withoutType() { - ArrayIdentifierExpression expr = ArrayIdentifierExpression.of("tags"); - String result = extractor.visit(expr); - assertNull(result); - } - - @Test - void testVisitJsonIdentifierExpression() { - JsonIdentifierExpression expr = JsonIdentifierExpression.of("customAttr", "field"); - assertThrows(UnsupportedOperationException.class, () -> extractor.visit(expr)); - } - - @Test - void testVisitIdentifierExpression() { - IdentifierExpression expr = IdentifierExpression.of("item"); - assertThrows(UnsupportedOperationException.class, () -> extractor.visit(expr)); - } - - @Test - void testVisitAggregateExpression() { - AggregateExpression expr = - AggregateExpression.of( - org.hypertrace.core.documentstore.expression.operators.AggregationOperator.COUNT, - IdentifierExpression.of("item")); - assertThrows(UnsupportedOperationException.class, () -> extractor.visit(expr)); - } - - @Test - void testVisitConstantExpression() { - ConstantExpression expr = ConstantExpression.of("test"); - assertThrows(UnsupportedOperationException.class, () -> extractor.visit(expr)); - } - - @Test - void testVisitDocumentConstantExpression() { - ConstantExpression.DocumentConstantExpression expr = - (ConstantExpression.DocumentConstantExpression) - ConstantExpression.of((org.hypertrace.core.documentstore.Document) null); - assertThrows(UnsupportedOperationException.class, () -> extractor.visit(expr)); - } - - @Test - void testVisitFunctionExpression() { - FunctionExpression expr = - FunctionExpression.builder() - .operator(FunctionOperator.LENGTH) - .operand(IdentifierExpression.of("item")) - .build(); - assertThrows(UnsupportedOperationException.class, () -> extractor.visit(expr)); - } - - @Test - void testVisitAliasedIdentifierExpression() { - AliasedIdentifierExpression expr = - AliasedIdentifierExpression.builder().name("item").contextAlias("i").build(); - assertThrows(UnsupportedOperationException.class, () -> extractor.visit(expr)); - } -} From 18df9d08a204428b9a0cf34bcc5721ff722007f1 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Thu, 18 Dec 2025 16:47:41 +0530 Subject: [PATCH 7/9] Added backward compat tests for IN operator on top-level array and scalar fields --- .../documentstore/DocStoreQueryV1Test.java | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java index cdefe083..a482c1ec 100644 --- a/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java +++ b/document-store/src/integrationTest/java/org/hypertrace/core/documentstore/DocStoreQueryV1Test.java @@ -4278,6 +4278,52 @@ void testNotInOperatorOnScalarFields(String dataStoreName) { long largeListNotInCount = flatCollection.count(largeListNotInQuery); assertEquals(7, largeListNotInCount); } + + @ParameterizedTest + @ArgumentsSource(PostgresProvider.class) + void testInOperatorWithoutTypeSpecified(String dataStoreName) { + Collection flatCollection = getFlatCollection(dataStoreName); + + // Test 1: IN operator on string field without type - uses fallback IN (?, ?, ?) syntax + Query untypedStringInQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("item"), + IN, + ConstantExpression.ofStrings(List.of("Soap", "Mirror", "Comb")))) + .build(); + + long untypedStringCount = flatCollection.count(untypedStringInQuery); + assertEquals(6, untypedStringCount); // 3 Soap + 1 Mirror + 2 Comb = 6 + + // Test 2: IN operator on integer field without type - uses fallback IN (?, ?, ?) syntax + Query untypedIntInQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("price"), + IN, + ConstantExpression.ofNumbers(List.of(5, 10, 15)))) + .build(); + + long untypedIntCount = flatCollection.count(untypedIntInQuery); + assertTrue(untypedIntCount > 0); + + // Test 3: NOT_IN operator on string field without type - uses fallback NOT IN (?, ?, ?) + // syntax + Query untypedNotInQuery = + Query.builder() + .setFilter( + RelationalExpression.of( + IdentifierExpression.of("item"), + NOT_IN, + ConstantExpression.ofStrings(List.of("Soap", "Mirror")))) + .build(); + + long untypedNotInCount = flatCollection.count(untypedNotInQuery); + assertEquals(6, untypedNotInCount); // 10 total - 3 Soap - 1 Mirror = 6 + } } @Nested @@ -5034,6 +5080,83 @@ void testAnyOnBooleanArray(String dataStoreName) throws IOException { assertDocsAndSizeEqualWithoutOrder( dataStoreName, resultIterator, "query/flat_boolean_array_filter_response.json", 5); } + + @ParameterizedTest + @ArgumentsSource(PostgresProvider.class) + void testInOperatorOnArrayFieldWithoutTypeSpecified(String dataStoreName) + throws JsonProcessingException { + Collection flatCollection = getFlatCollection(dataStoreName); + + // Test 1: IN operator on string array field without type - uses fallback syntax + Query untypedStringArrayInQuery = + Query.builder() + .addSelection(IdentifierExpression.of("item")) + .addSelection(ArrayIdentifierExpression.of("tags")) + .setFilter( + RelationalExpression.of( + ArrayIdentifierExpression.of("tags"), + IN, + ConstantExpression.ofStrings(List.of("hygiene", "grooming")))) + .build(); + + Iterator stringResults = flatCollection.find(untypedStringArrayInQuery); + int stringCount = 0; + while (stringResults.hasNext()) { + Document doc = stringResults.next(); + JsonNode json = new ObjectMapper().readTree(doc.toJson()); + stringCount++; + + // Verify that returned arrays contain at least one of the IN values + JsonNode tags = json.get("tags"); + if (tags != null && tags.isArray()) { + boolean containsMatch = false; + for (JsonNode tag : tags) { + String tagValue = tag.asText(); + if ("hygiene".equals(tagValue) || "grooming".equals(tagValue)) { + containsMatch = true; + break; + } + } + assertTrue(containsMatch, "Array should contain at least one IN value"); + } + } + assertTrue(stringCount >= 5, "Should return at least 5 items with hygiene/grooming tags"); + + // Test 2: IN operator on integer array field without type - uses fallback syntax + Query untypedIntArrayInQuery = + Query.builder() + .addSelection(IdentifierExpression.of("item")) + .addSelection(ArrayIdentifierExpression.of("numbers")) + .setFilter( + RelationalExpression.of( + ArrayIdentifierExpression.of("numbers"), + IN, + ConstantExpression.ofNumbers(List.of(1, 10, 20)))) + .build(); + + Iterator intResults = flatCollection.find(untypedIntArrayInQuery); + int intCount = 0; + while (intResults.hasNext()) { + Document doc = intResults.next(); + JsonNode json = new ObjectMapper().readTree(doc.toJson()); + intCount++; + + // Verify that returned arrays contain at least one of the IN values + JsonNode numbers = json.get("numbers"); + if (numbers != null && numbers.isArray()) { + boolean containsMatch = false; + for (JsonNode num : numbers) { + int value = num.asInt(); + if (value == 1 || value == 10 || value == 20) { + containsMatch = true; + break; + } + } + assertTrue(containsMatch, "Array should contain at least one IN value"); + } + } + assertTrue(intCount >= 6, "Should return at least 6 items"); + } } @Nested From b59956462f8ae441d6fd3889a5edab613b1f8142 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Thu, 18 Dec 2025 17:00:20 +0530 Subject: [PATCH 8/9] Fix failing test case --- .../field/PostgresInRelationalFilterParserArrayField.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java index 20b2f093..16095b8c 100644 --- a/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java +++ b/document-store/src/main/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresInRelationalFilterParserArrayField.java @@ -97,8 +97,9 @@ private String prepareFilterStringForArrayInOperator( paramsBuilder.addArrayParam(values, sqlType); return String.format("%s && ?", parsedLhs); } else { - // Fallback: use jsonb array contains approach - return prepareFilterStringFallback(parsedLhs, parsedRhs, paramsBuilder, "%s @> ARRAY[%s]"); + // Fallback: cast both sides to text[] for backward compatibility with any array type + return prepareFilterStringFallback( + parsedLhs, parsedRhs, paramsBuilder, "%s::text[] && ARRAY[%s]::text[]"); } } From ffed7249f7231b849936905b769855cec51e6485 Mon Sep 17 00:00:00 2001 From: Prashant Pandey Date: Thu, 18 Dec 2025 17:12:55 +0530 Subject: [PATCH 9/9] Added PostgresTypeExtractorTest --- .../field/PostgresTypeExtractorTest.java | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresTypeExtractorTest.java diff --git a/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresTypeExtractorTest.java b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresTypeExtractorTest.java new file mode 100644 index 00000000..6d4e59f6 --- /dev/null +++ b/document-store/src/test/java/org/hypertrace/core/documentstore/postgres/query/v1/parser/filter/nonjson/field/PostgresTypeExtractorTest.java @@ -0,0 +1,132 @@ +package org.hypertrace.core.documentstore.postgres.query.v1.parser.filter.nonjson.field; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.hypertrace.core.documentstore.expression.impl.AggregateExpression; +import org.hypertrace.core.documentstore.expression.impl.AliasedIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.ArrayIdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.ConstantExpression; +import org.hypertrace.core.documentstore.expression.impl.FunctionExpression; +import org.hypertrace.core.documentstore.expression.impl.IdentifierExpression; +import org.hypertrace.core.documentstore.expression.impl.JsonIdentifierExpression; +import org.hypertrace.core.documentstore.expression.operators.FunctionOperator; +import org.junit.jupiter.api.Test; + +class PostgresTypeExtractorTest { + + private final PostgresTypeExtractor extractor = PostgresTypeExtractor.arrayType(); + + @Test + void testVisitArrayIdentifierExpression_withType() { + ArrayIdentifierExpression expr = ArrayIdentifierExpression.ofStrings("tags"); + String result = extractor.visit(expr); + assertEquals("text[]", result); + } + + @Test + void testVisitArrayIdentifierExpression_withIntegerType() { + ArrayIdentifierExpression expr = ArrayIdentifierExpression.ofInts("numbers"); + String result = extractor.visit(expr); + assertEquals("int4[]", result); + } + + @Test + void testVisitArrayIdentifierExpression_withBooleanType() { + ArrayIdentifierExpression expr = ArrayIdentifierExpression.ofBooleans("flags"); + String result = extractor.visit(expr); + assertEquals("bool[]", result); + } + + @Test + void testVisitArrayIdentifierExpression_withoutType() { + ArrayIdentifierExpression expr = ArrayIdentifierExpression.of("tags"); + String result = extractor.visit(expr); + assertNull(result); + } + + @Test + void testVisitJsonIdentifierExpression() { + JsonIdentifierExpression expr = JsonIdentifierExpression.of("customAttr", "field"); + assertThrows(UnsupportedOperationException.class, () -> extractor.visit(expr)); + } + + @Test + void testVisitIdentifierExpression_withType() { + IdentifierExpression expr = IdentifierExpression.ofString("item"); + String result = extractor.visit(expr); + assertEquals("text[]", result); + } + + @Test + void testVisitIdentifierExpression_withoutType() { + IdentifierExpression expr = IdentifierExpression.of("item"); + String result = extractor.visit(expr); + assertNull(result); + } + + @Test + void testVisitIdentifierExpression_scalarType_withStringType() { + PostgresTypeExtractor scalarExtractor = PostgresTypeExtractor.scalarType(); + IdentifierExpression expr = IdentifierExpression.ofString("item"); + String result = scalarExtractor.visit(expr); + assertEquals("text", result); + } + + @Test + void testVisitIdentifierExpression_scalarType_withIntType() { + PostgresTypeExtractor scalarExtractor = PostgresTypeExtractor.scalarType(); + IdentifierExpression expr = IdentifierExpression.ofInt("price"); + String result = scalarExtractor.visit(expr); + assertEquals("int4", result); + } + + @Test + void testVisitIdentifierExpression_scalarType_withoutType() { + PostgresTypeExtractor scalarExtractor = PostgresTypeExtractor.scalarType(); + IdentifierExpression expr = IdentifierExpression.of("item"); + String result = scalarExtractor.visit(expr); + assertNull(result); + } + + @Test + void testVisitAggregateExpression() { + AggregateExpression expr = + AggregateExpression.of( + org.hypertrace.core.documentstore.expression.operators.AggregationOperator.COUNT, + IdentifierExpression.of("item")); + assertThrows(UnsupportedOperationException.class, () -> extractor.visit(expr)); + } + + @Test + void testVisitConstantExpression() { + ConstantExpression expr = ConstantExpression.of("test"); + assertThrows(UnsupportedOperationException.class, () -> extractor.visit(expr)); + } + + @Test + void testVisitDocumentConstantExpression() { + ConstantExpression.DocumentConstantExpression expr = + (ConstantExpression.DocumentConstantExpression) + ConstantExpression.of((org.hypertrace.core.documentstore.Document) null); + assertThrows(UnsupportedOperationException.class, () -> extractor.visit(expr)); + } + + @Test + void testVisitFunctionExpression() { + FunctionExpression expr = + FunctionExpression.builder() + .operator(FunctionOperator.LENGTH) + .operand(IdentifierExpression.of("item")) + .build(); + assertThrows(UnsupportedOperationException.class, () -> extractor.visit(expr)); + } + + @Test + void testVisitAliasedIdentifierExpression() { + AliasedIdentifierExpression expr = + AliasedIdentifierExpression.builder().name("item").contextAlias("i").build(); + assertThrows(UnsupportedOperationException.class, () -> extractor.visit(expr)); + } +}