From b42ad3ca70d2b54e69024c23bb8b114efcbc9201 Mon Sep 17 00:00:00 2001 From: Daniel Belchior Date: Fri, 31 Jul 2026 17:22:59 +0200 Subject: [PATCH 01/18] Add query.max-output-data-size configuration property Add `query.max-output-data-size` configuration property Enforce a maximum limit on the output data size a query can produce during execution. If a query exceeds this limit, execution is terminated with the `EXCEEDED_OUTPUT_LIMIT` error code. --- .../trino/ExceededOutputLimitException.java | 28 +++++++++++++++++++ .../java/io/trino/execution/QueryManager.java | 21 ++++++++++++++ .../trino/execution/QueryManagerConfig.java | 14 ++++++++++ .../execution/TestQueryManagerConfig.java | 3 ++ .../java/io/trino/spi/StandardErrorCode.java | 1 + .../admin/properties-query-management.md | 8 ++++++ 6 files changed, 75 insertions(+) create mode 100644 core/trino-main/src/main/java/io/trino/ExceededOutputLimitException.java diff --git a/core/trino-main/src/main/java/io/trino/ExceededOutputLimitException.java b/core/trino-main/src/main/java/io/trino/ExceededOutputLimitException.java new file mode 100644 index 000000000000..8cf830303de2 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/ExceededOutputLimitException.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino; + +import io.airlift.units.DataSize; +import io.trino.spi.TrinoException; + +import static io.trino.spi.StandardErrorCode.EXCEEDED_OUTPUT_LIMIT; + +public class ExceededOutputLimitException + extends TrinoException +{ + public ExceededOutputLimitException(DataSize limit) + { + super(EXCEEDED_OUTPUT_LIMIT, "Exceeded output data size limit of " + limit); + } +} diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryManager.java b/core/trino-main/src/main/java/io/trino/execution/QueryManager.java index 03c5abae1b54..c62da9a7c102 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryManager.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryManager.java @@ -24,6 +24,7 @@ import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Context; import io.trino.ExceededCpuLimitException; +import io.trino.ExceededOutputLimitException; import io.trino.ExceededScanLimitException; import io.trino.ExceededWriteLimitException; import io.trino.Session; @@ -80,6 +81,7 @@ public class QueryManager private final Duration maxQueryCpuTime; private final Optional maxQueryScanPhysicalBytes; private final Optional maxQueryWritePhysicalSize; + private final Optional maxQueryOutputDataSize; private final ExecutorService queryExecutor; private final ThreadPoolExecutorMBean queryExecutorMBean; @@ -96,6 +98,7 @@ public QueryManager(ClusterMemoryManager memoryManager, Tracer tracer, QueryMana this.maxQueryCpuTime = queryManagerConfig.getQueryMaxCpuTime(); this.maxQueryScanPhysicalBytes = queryManagerConfig.getQueryMaxScanPhysicalBytes(); this.maxQueryWritePhysicalSize = queryManagerConfig.getQueryMaxWritePhysicalSize(); + this.maxQueryOutputDataSize = queryManagerConfig.getQueryMaxOutputDataSize(); this.queryExecutor = newCachedThreadPool(threadsNamed("query-scheduler-%s")); this.queryExecutorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) queryExecutor); @@ -138,6 +141,13 @@ public void start() catch (Throwable e) { log.error(e, "Error enforcing query write bytes limits"); } + + try { + enforceOutputDataSizeLimits(); + } + catch (Throwable e) { + log.error(e, "Error enforcing query output data size limits"); + } }, 1, 1, TimeUnit.SECONDS); } @@ -456,4 +466,15 @@ private void enforceWriteLimits() }); } } + + private void enforceOutputDataSizeLimits() + { + maxQueryOutputDataSize.ifPresent(outputLimit -> { + for (QueryExecution query : queryTracker.getAllQueries()) { + if (query.getQueryInfo().getQueryStats().getOutputDataSize().compareTo(outputLimit) > 0) { + query.fail(new ExceededOutputLimitException(outputLimit)); + } + } + }); + } } diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java b/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java index 0bdfaaee722b..d8a42c71bddf 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java @@ -103,6 +103,7 @@ public class QueryManagerConfig private Duration queryMaxCpuTime = new Duration(1_000_000_000, TimeUnit.DAYS); private Optional queryMaxScanPhysicalBytes = Optional.empty(); private Optional queryMaxWritePhysicalSize = Optional.empty(); + private Optional queryMaxOutputDataSize = Optional.empty(); private int queryReportedRuleStatsLimit = 10; private int dispatcherQueryPoolSize = DISPATCHER_THREADPOOL_MAX_SIZE; @@ -542,6 +543,19 @@ public QueryManagerConfig setQueryMaxWritePhysicalSize(DataSize queryMaxWritePhy return this; } + @NotNull + public Optional<@MinDataSize("1MB") DataSize> getQueryMaxOutputDataSize() + { + return queryMaxOutputDataSize; + } + + @Config("query.max-output-data-size") + public QueryManagerConfig setQueryMaxOutputDataSize(DataSize queryMaxOutputDataSize) + { + this.queryMaxOutputDataSize = Optional.ofNullable(queryMaxOutputDataSize); + return this; + } + @Min(1) public int getQueryReportedRuleStatsLimit() { diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java index b85aea2ec6d1..0105dabbc30c 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java @@ -77,6 +77,7 @@ public void testDefaults() .setDispatcherQueryPoolSize(Integer.toString(max(50, Runtime.getRuntime().availableProcessors() * 10))) .setQueryMaxScanPhysicalBytes(null) .setQueryMaxWritePhysicalSize(null) + .setQueryMaxOutputDataSize(null) .setRequiredWorkers(1) .setRequiredWorkersMaxWait(new Duration(5, MINUTES)) .setRetryPolicy(RetryPolicy.NONE) @@ -164,6 +165,7 @@ public void testExplicitPropertyMappings() .put("query.dispatcher-query-pool-size", "151") .put("query.max-scan-physical-bytes", "1kB") .put("query.max-write-physical-size", "1TB") + .put("query.max-output-data-size", "1TB") .put("query-manager.required-workers", "333") .put("query-manager.required-workers-max-wait", "33m") .put("retry-policy", "QUERY") @@ -248,6 +250,7 @@ public void testExplicitPropertyMappings() .setDispatcherQueryPoolSize("151") .setQueryMaxScanPhysicalBytes(DataSize.of(1, KILOBYTE)) .setQueryMaxWritePhysicalSize(DataSize.of(1, TERABYTE)) + .setQueryMaxOutputDataSize(DataSize.of(1, TERABYTE)) .setRequiredWorkers(333) .setRequiredWorkersMaxWait(new Duration(33, MINUTES)) .setRetryPolicy(RetryPolicy.QUERY) diff --git a/core/trino-spi/src/main/java/io/trino/spi/StandardErrorCode.java b/core/trino-spi/src/main/java/io/trino/spi/StandardErrorCode.java index 8f75da5fcd40..d0faec5bce5d 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/StandardErrorCode.java +++ b/core/trino-spi/src/main/java/io/trino/spi/StandardErrorCode.java @@ -206,6 +206,7 @@ public enum StandardErrorCode EXCEEDED_SCAN_LIMIT(131081, INSUFFICIENT_RESOURCES), EXCEEDED_TASK_DESCRIPTOR_STORAGE_CAPACITY(131082, INSUFFICIENT_RESOURCES), EXCEEDED_WRITE_LIMIT(131083, INSUFFICIENT_RESOURCES), + EXCEEDED_OUTPUT_LIMIT(131084, INSUFFICIENT_RESOURCES), UNSUPPORTED_TABLE_TYPE(133001, EXTERNAL), /**/; diff --git a/docs/src/main/sphinx/admin/properties-query-management.md b/docs/src/main/sphinx/admin/properties-query-management.md index 9a90d78f5fa6..e2561ea0c56a 100644 --- a/docs/src/main/sphinx/admin/properties-query-management.md +++ b/docs/src/main/sphinx/admin/properties-query-management.md @@ -168,6 +168,14 @@ The maximum physical size of data that can be written by a query during its exec When this limit is reached, query processing is terminated to prevent excessive resource usage. +## `query.max-output-data-size` + +- **Type:** {ref}`prop-type-data-size` + +The maximum size of the output data that a query can produce during its execution. +When this limit is exceeded, query processing is terminated. This limit protects +against queries that return excessively large result sets to the client. + ## `query.max-stage-count` - **Type:** {ref}`prop-type-integer` From 5e4b390a28e48e96869f69dcba5dce40f340047c Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Fri, 4 Sep 2026 17:55:22 +0900 Subject: [PATCH 02/18] Fix deprecated warning in Iceberg --- .../io/trino/plugin/iceberg/IcebergFileWriterFactory.java | 8 +++++++- .../io/trino/plugin/iceberg/TestIcebergSplitSource.java | 7 +------ .../test/java/io/trino/plugin/iceberg/TestIcebergV2.java | 6 +----- .../test/java/io/trino/plugin/iceberg/TestIcebergV3.java | 2 +- .../trino/plugin/iceberg/util/EncryptedFileTestUtils.java | 2 +- 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java index 544f5ffefd48..ac689db4a10a 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java @@ -351,7 +351,13 @@ private static Types.NestedField toFileType(Types.NestedField field) if (type == field.type()) { return field; } - return Types.NestedField.of(field.fieldId(), field.isOptional(), field.name(), type, field.doc()); + return Types.NestedField.builder() + .withId(field.fieldId()) + .isOptional(field.isOptional()) + .withName(field.name()) + .ofType(type) + .withDoc(field.doc()) + .build(); } private static org.apache.iceberg.types.Type toFileType(org.apache.iceberg.types.Type type) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java index 5bb036fc9b32..43eca65d2f42 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSplitSource.java @@ -52,9 +52,7 @@ import org.apache.iceberg.SchemaParser; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; -import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; -import org.apache.iceberg.data.parquet.GenericParquetWriter; import org.apache.iceberg.deletes.PositionDelete; import org.apache.iceberg.deletes.PositionDeleteWriter; import org.apache.iceberg.encryption.EncryptedInputFile; @@ -333,14 +331,11 @@ public void testSplitWeight() // Write position delete file FileIO fileIo = FILE_IO_FACTORY.create(fileSystemFactory.create(SESSION)); PositionDeleteWriter writer = Parquet.writeDeletes(fileIo.newOutputFile("local:///delete_file_" + UUID.randomUUID())) - .createWriterFunc(GenericParquetWriter::create) - .forTable(nationTable) .overwrite() - .rowSchema(nationTable.schema()) .withSpec(PartitionSpec.unpartitioned()) .buildPositionWriter(); PositionDelete positionDelete = PositionDelete.create(); - PositionDelete record = positionDelete.set(dataFilePath, 0, GenericRecord.create(nationTable.schema())); + PositionDelete record = positionDelete.set(dataFilePath, 0); try (Closeable ignored = writer) { writer.write(record); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java index ac7ecc803f78..ab7fa1b04a4b 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java @@ -65,7 +65,6 @@ import org.apache.iceberg.TableProperties; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; -import org.apache.iceberg.data.parquet.GenericParquetWriter; import org.apache.iceberg.data.parquet.InternalWriter; import org.apache.iceberg.deletes.PositionDelete; import org.apache.iceberg.deletes.PositionDeleteWriter; @@ -286,15 +285,12 @@ public void testV2TableWithPositionDelete() FileIO fileIo = FILE_IO_FACTORY.create(fileSystemFactory.create(SESSION)); PositionDeleteWriter writer = Parquet.writeDeletes(fileIo.newOutputFile("local:///delete_file_" + UUID.randomUUID())) - .createWriterFunc(GenericParquetWriter::create) - .forTable(icebergTable) .overwrite() - .rowSchema(icebergTable.schema()) .withSpec(PartitionSpec.unpartitioned()) .buildPositionWriter(); PositionDelete positionDelete = PositionDelete.create(); - PositionDelete record = positionDelete.set(dataFilePath, 0, GenericRecord.create(icebergTable.schema())); + PositionDelete record = positionDelete.set(dataFilePath, 0); try (Closeable ignored = writer) { writer.write(record); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV3.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV3.java index 4a2bfb216b44..61792cb09184 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV3.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV3.java @@ -1599,7 +1599,7 @@ private ParquetMetadata getOnlyParquetDataFileMetadata(String tableName) { BaseTable table = loadTable(tableName); table.refresh(); - DataFile dataFile = getOnlyElement(table.currentSnapshot().addedDataFiles(table.io())); + DataFile dataFile = getOnlyElement(SnapshotChanges.builderFor(table).build().addedDataFiles()); return getParquetFileMetadata(fileSystemFactory.create(SESSION).newInputFile(Location.of(dataFile.location()))); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/util/EncryptedFileTestUtils.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/util/EncryptedFileTestUtils.java index d1cc68f60c2f..0d334296f905 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/util/EncryptedFileTestUtils.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/util/EncryptedFileTestUtils.java @@ -213,7 +213,7 @@ public static DataFile writePlaintextFileWithEncryptionKeyMetadata( DataFile plaintextFile = writer.toDataFile(); return DataFiles.builder(table.spec()) - .withPath(plaintextFile.path().toString()) + .withPath(plaintextFile.location()) .withFileSizeInBytes(plaintextFile.fileSizeInBytes()) .withRecordCount(plaintextFile.recordCount()) .withFormat(plaintextFile.format()) From 7b79356987162010552ecac691c62c0d8141be43 Mon Sep 17 00:00:00 2001 From: bvolpato Date: Thu, 3 Sep 2026 19:30:41 +0000 Subject: [PATCH 03/18] Fix decimal average overflow scaling Use the input scale for the overflow contribution when reconstructing the decimal sum. Previously only the wrapped 128-bit value was scaled, leading to incorrect averages or spurious overflow failures. --- .../DecimalAverageAggregation.java | 2 +- .../TestDecimalAverageAggregation.java | 23 ++++++++++++++++ .../io/trino/sql/query/TestAggregation.java | 27 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/core/trino-main/src/main/java/io/trino/operator/aggregation/DecimalAverageAggregation.java b/core/trino-main/src/main/java/io/trino/operator/aggregation/DecimalAverageAggregation.java index 39b9161b308b..28be00812047 100644 --- a/core/trino-main/src/main/java/io/trino/operator/aggregation/DecimalAverageAggregation.java +++ b/core/trino-main/src/main/java/io/trino/operator/aggregation/DecimalAverageAggregation.java @@ -160,7 +160,7 @@ public static Int128 average(LongDecimalWithOverflowAndLongState state, DecimalT long overflow = state.getOverflow(); if (overflow != 0) { BigDecimal sum = new BigDecimal(Int128.valueOf(decimal[offset], decimal[offset + 1]).toBigInteger(), type.getScale()); - sum = sum.add(new BigDecimal(OVERFLOW_MULTIPLIER.multiply(BigInteger.valueOf(overflow)))); + sum = sum.add(new BigDecimal(OVERFLOW_MULTIPLIER.multiply(BigInteger.valueOf(overflow)), type.getScale())); BigDecimal count = BigDecimal.valueOf(state.getLong()); return Decimals.encodeScaledValue(sum.divide(count, type.getScale(), HALF_UP), type.getScale()); diff --git a/core/trino-main/src/test/java/io/trino/operator/aggregation/TestDecimalAverageAggregation.java b/core/trino-main/src/test/java/io/trino/operator/aggregation/TestDecimalAverageAggregation.java index 8897405f6150..3f3eb3810dfc 100644 --- a/core/trino-main/src/test/java/io/trino/operator/aggregation/TestDecimalAverageAggregation.java +++ b/core/trino-main/src/test/java/io/trino/operator/aggregation/TestDecimalAverageAggregation.java @@ -22,6 +22,8 @@ import io.trino.spi.type.Decimals; import io.trino.spi.type.Int128; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.math.BigDecimal; import java.math.BigInteger; @@ -162,6 +164,27 @@ public void testCombineUnderflow() assertAverageEquals(state, expectedAverage); } + @ParameterizedTest + @ValueSource(ints = {1, 2, 18, 38}) + public void testOverflowWithScale(int scale) + { + DecimalType type = createDecimalType(38, scale); + BigInteger value = Decimals.MAX_UNSCALED_DECIMAL.toBigInteger(); + for (BigInteger number : ImmutableList.of(value, value.negate())) { + LongDecimalWithOverflowAndLongState state = new LongDecimalWithOverflowAndLongStateFactory().createSingleState(); + addToState(type, state, number); + addToState(type, state, number); + assertThat(state.getOverflow()).isNotZero(); + assertThat(average(state, type).toBigInteger()).isEqualTo(number); + + LongDecimalWithOverflowAndLongState otherState = new LongDecimalWithOverflowAndLongStateFactory().createSingleState(); + addToState(type, otherState, number); + addToState(type, otherState, number); + DecimalAverageAggregation.combine(state, otherState); + assertThat(average(state, type).toBigInteger()).isEqualTo(number); + } + } + @Test public void testNoOverflow() { diff --git a/core/trino-main/src/test/java/io/trino/sql/query/TestAggregation.java b/core/trino-main/src/test/java/io/trino/sql/query/TestAggregation.java index a8b9c6176d92..dd55c887d1b5 100644 --- a/core/trino-main/src/test/java/io/trino/sql/query/TestAggregation.java +++ b/core/trino-main/src/test/java/io/trino/sql/query/TestAggregation.java @@ -40,6 +40,33 @@ public void teardown() assertions.close(); } + @Test + public void testDecimalAverageWithScaledOverflow() + { + assertThat(assertions.query("SELECT avg(x) FROM UNNEST(CAST(ARRAY[0.9, 0.9] AS array(decimal(38,38)))) t(x)")) + .matches("VALUES CAST(0.9 AS decimal(38,38))"); + } + + @Test + public void testDecimalAverageWithoutOverflow() + { + assertThat(assertions.query("SELECT avg(x) FROM UNNEST(CAST(ARRAY[0.8, 0.8] AS array(decimal(38,38)))) t(x)")) + .matches("VALUES CAST(0.8 AS decimal(38,38))"); + } + + @Test + public void testDecimalAverageWithScaledOverflowAndZeroValues() + { + assertThat(assertions.query( + """ + SELECT avg(x) + FROM UNNEST(concat( + ARRAY[DECIMAL '9000000000000000000000000000000000000.0', DECIMAL '9000000000000000000000000000000000000.0'], + repeat(CAST(0 AS decimal(38,1)), 98))) t(x) + """)) + .matches("VALUES CAST(DECIMAL '180000000000000000000000000000000000.0' AS decimal(38,1))"); + } + @Test public void testQuantifiedComparison() { From 0dac4acfd29ff30a5540881d0ce67838be502c6c Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Fri, 4 Sep 2026 14:00:05 +0200 Subject: [PATCH 04/18] Update endive to 1.1.0 --- plugin/trino-functions-python/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/trino-functions-python/pom.xml b/plugin/trino-functions-python/pom.xml index a468baf355a4..ca92b1881fb3 100644 --- a/plugin/trino-functions-python/pom.xml +++ b/plugin/trino-functions-python/pom.xml @@ -18,7 +18,7 @@ run.endive bom - 1.0.1 + 1.1.0 pom import From 19b4cebac145d9d1fec3d202df6feefebd6e8c50 Mon Sep 17 00:00:00 2001 From: "Mateusz \"Serafin\" Gajewski" Date: Fri, 4 Sep 2026 13:56:53 +0200 Subject: [PATCH 05/18] Upgrade to JLine 4.4.2 and remove workaround --- client/trino-cli/pom.xml | 2 +- .../main/java/io/trino/cli/StatusPrinter.java | 23 ------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/client/trino-cli/pom.xml b/client/trino-cli/pom.xml index 2dbd40f34602..e3479150c7ed 100644 --- a/client/trino-cli/pom.xml +++ b/client/trino-cli/pom.xml @@ -18,7 +18,7 @@ false UnusedLambdaParameterShouldBeUnnamed,UseEnhancedSwitch io.trino.cli.Trino - 4.4.1 + 4.4.2 diff --git a/client/trino-cli/src/main/java/io/trino/cli/StatusPrinter.java b/client/trino-cli/src/main/java/io/trino/cli/StatusPrinter.java index 3a69d4072f47..4ad050b27aa4 100644 --- a/client/trino-cli/src/main/java/io/trino/cli/StatusPrinter.java +++ b/client/trino-cli/src/main/java/io/trino/cli/StatusPrinter.java @@ -22,15 +22,10 @@ import io.trino.client.StatementStats; import org.jline.terminal.Attributes; import org.jline.terminal.Terminal; -import org.jline.terminal.impl.AbstractUnixSysTerminal; import org.jline.utils.AttributedString; import org.jline.utils.AttributedStyle; -import org.jline.utils.NonBlockingReader; -import java.io.FileDescriptor; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.PrintStream; import java.util.List; import java.util.OptionalInt; @@ -64,16 +59,6 @@ public class StatusPrinter private static final int CTRL_C = 3; private static final int CTRL_P = 16; - // A timed read on terminal.reader() that expires leaves JLine's pump thread blocked - // in a read on the tty, where it steals the next keystroke typed into an external - // process spawned afterwards, such as the pager. None of the JLine stream wrappers - // report available(), so poll the stdin file descriptor directly and only issue a - // read once a byte is queued, so that it completes without parking the pump thread. - // This only applies to terminals reading the process stdin: AbstractUnixSysTerminal - // is created only when stdin is a tty and always reads FileDescriptor.in. Other - // terminals (Windows console, /dev/tty fallback, dumb) keep the timed read. - private static final InputStream RAW_TERMINAL_INPUT = new FileInputStream(FileDescriptor.in); - private final long start = System.nanoTime(); private final StatementClient client; private final PrintStream out; @@ -486,14 +471,6 @@ private void reprintLine(String line) private static int readKey(Terminal terminal) { try { - if (terminal instanceof AbstractUnixSysTerminal) { - if (RAW_TERMINAL_INPUT.available() == 0) { - return NonBlockingReader.READ_EXPIRED; - } - // a byte is queued, so the read returns immediately; the generous timeout - // only covers pump thread scheduling delays - return terminal.reader().read(100L); - } return terminal.reader().read(1L); } catch (IOException e) { From 90589d7b453feccaf569424cd5985b2494a38377 Mon Sep 17 00:00:00 2001 From: Ray Cheng Date: Wed, 2 Sep 2026 15:54:34 +0800 Subject: [PATCH 06/18] Use Kerberos image for Hive product tests --- .../product/hive/HiveKerberosEnvironment.java | 214 +++++------------- 1 file changed, 54 insertions(+), 160 deletions(-) diff --git a/testing/trino-product-tests/src/test/java/io/trino/tests/product/hive/HiveKerberosEnvironment.java b/testing/trino-product-tests/src/test/java/io/trino/tests/product/hive/HiveKerberosEnvironment.java index 72a87dd0c158..77375a1c0857 100644 --- a/testing/trino-product-tests/src/test/java/io/trino/tests/product/hive/HiveKerberosEnvironment.java +++ b/testing/trino-product-tests/src/test/java/io/trino/tests/product/hive/HiveKerberosEnvironment.java @@ -43,18 +43,17 @@ * This environment provides: *
    *
  • Standalone KDC container for Kerberos authentication
    • - *
  • Hadoop container (HDFS, Hive Metastore) configured for Kerberos via init script injection
    • + *
  • Hadoop container (HDFS, Hive Metastore) configured for Kerberos
    • *
  • Trino container with Kerberos-enabled Hive connector
    • *
*

* Implementation: *

- * The Hadoop image has a built-in init hook mechanism where scripts in - * {@code /etc/hadoop-init.d/} are executed before supervisord starts. This environment - * injects a Kerberos configuration script that: + * The Hadoop image provides the standard Kerberos client tooling and Hadoop/Hive configuration. + * It also has a built-in init hook mechanism where scripts in {@code /etc/hadoop-init.d/} + * are executed before supervisord starts. This environment injects a script that: *

    - *
  • Ensures krb5-workstation Kerberos tools are available (skips install when already present)
    • - *
  • Modifies Hadoop/Hive configuration files to enable Kerberos authentication
    • + *
  • Applies environment-specific Hadoop/Hive configuration overrides
    • *
*

* Architecture: @@ -83,6 +82,8 @@ *

  • hdfs/hadoop-master@TRINO.TEST - for HDFS services
    • *
  • hive/hadoop-master@TRINO.TEST - for Hive Metastore
    • *
  • HTTP/hadoop-master@TRINO.TEST - for WebHDFS/HTTP SPNEGO
    • + *
  • mapred/hadoop-master@TRINO.TEST - for MapReduce services
    • + *
  • yarn/hadoop-master@TRINO.TEST - for YARN services
    • *
  • trino/trino-master@TRINO.TEST - for Trino service
    • * *

    @@ -113,19 +114,21 @@ public class HiveKerberosEnvironment protected static final String HDFS_PRINCIPAL = "hdfs/hadoop-master"; protected static final String HIVE_PRINCIPAL = "hive/hadoop-master"; protected static final String HTTP_PRINCIPAL = "HTTP/hadoop-master"; + protected static final String MAPRED_PRINCIPAL = "mapred/hadoop-master"; + protected static final String YARN_PRINCIPAL = "yarn/hadoop-master"; protected static final String TRINO_PRINCIPAL = "trino/trino-master"; // Keytab paths in KDC container protected static final String KDC_HDFS_KEYTAB_PATH = "/keytabs/hdfs.keytab"; protected static final String KDC_HIVE_KEYTAB_PATH = "/keytabs/hive.keytab"; - protected static final String KDC_HTTP_KEYTAB_PATH = "/keytabs/http.keytab"; + protected static final String KDC_HTTP_KEYTAB_PATH = "/keytabs/HTTP.keytab"; + protected static final String KDC_MAPRED_KEYTAB_PATH = "/keytabs/mapred.keytab"; + protected static final String KDC_YARN_KEYTAB_PATH = "/keytabs/yarn.keytab"; protected static final String KDC_TRINO_KEYTAB_PATH = "/keytabs/trino.keytab"; // Paths where keytabs are mounted in Hadoop container protected static final String HADOOP_KEYTAB_DIR = "/etc/security/keytabs"; protected static final String HADOOP_HDFS_KEYTAB = HADOOP_KEYTAB_DIR + "/hdfs.keytab"; - protected static final String HADOOP_HIVE_KEYTAB = HADOOP_KEYTAB_DIR + "/hive.keytab"; - protected static final String HADOOP_HTTP_KEYTAB = HADOOP_KEYTAB_DIR + "/http.keytab"; // Paths in Trino container protected static final String TRINO_KEYTAB = "/etc/trino/trino.keytab"; @@ -173,6 +176,8 @@ public void start() .withPrincipal(HDFS_PRINCIPAL, KDC_HDFS_KEYTAB_PATH) .withPrincipal(HIVE_PRINCIPAL, KDC_HIVE_KEYTAB_PATH) .withPrincipal(HTTP_PRINCIPAL, KDC_HTTP_KEYTAB_PATH) + .withPrincipal(MAPRED_PRINCIPAL, KDC_MAPRED_KEYTAB_PATH) + .withPrincipal(YARN_PRINCIPAL, KDC_YARN_KEYTAB_PATH) .withPrincipal(TRINO_PRINCIPAL, KDC_TRINO_KEYTAB_PATH); // Add any additional principals from subclasses @@ -445,7 +450,9 @@ private void writeKerberosFiles() // When bind-mounted, files keep host permissions, so we make them world-readable writeKeytab(keytabDir, "hdfs.keytab", KDC_HDFS_KEYTAB_PATH); writeKeytab(keytabDir, "hive.keytab", KDC_HIVE_KEYTAB_PATH); - writeKeytab(keytabDir, "http.keytab", KDC_HTTP_KEYTAB_PATH); + writeKeytab(keytabDir, "HTTP.keytab", KDC_HTTP_KEYTAB_PATH); + writeKeytab(keytabDir, "mapred.keytab", KDC_MAPRED_KEYTAB_PATH); + writeKeytab(keytabDir, "yarn.keytab", KDC_YARN_KEYTAB_PATH); writeKeytab(keytabDir, "trino.keytab", KDC_TRINO_KEYTAB_PATH); // Allow subclasses to write additional keytabs @@ -486,11 +493,6 @@ private HadoopContainer createKerberosHadoopContainer() .withNetwork(network) .withNetworkAliases(HadoopContainer.HOST_NAME); - // Set JAVA_TOOL_OPTIONS so all JVM processes can find krb5.conf - // This is necessary because supervisord-started services don't inherit - // environment from the init script, and this ensures consistent Kerberos config - container.withEnv("JAVA_TOOL_OPTIONS", "-Djava.security.krb5.conf=/etc/krb5.conf"); - // Bind mount Kerberos files so they're available when entrypoint runs container.withFileSystemBind( tempDir.resolve("krb5.conf").toString(), @@ -513,172 +515,64 @@ private HadoopContainer createKerberosHadoopContainer() /** * Extension point for selecting the Hadoop image used by Kerberos environments. - * Default preserves launcher-parity base image and script-injection configuration. - * Subclasses can opt into pre-baked image variants. + * Subclasses can override this to select a different pre-baked image variant. */ protected HadoopContainer createKerberosBaseHadoopContainer() { - return new HadoopContainer(); + return HadoopContainer.kerberized(); } /** * Generates the Kerberos initialization script that runs before supervisord. *

    - * This script: - *

      - *
    • Ensures krb5-workstation tools are available (kinit, klist, etc.)
      • - *
    • Modifies Hadoop configuration files to enable Kerberos authentication
      • - *
    • Modifies Hive configuration files to enable Kerberos for Metastore
      • - *
    + * The image owns the standard Kerberos configuration. This script only applies + * properties supplied by specialized product-test environments. */ private String generateKerberosInitScript() { - String realm = kdc.getRealm(); + Map hdfsSiteProperties = new LinkedHashMap<>(); + hdfsSiteProperties.put("dfs.client.use.datanode.hostname", "true"); + hdfsSiteProperties.put("dfs.datanode.use.datanode.hostname", "true"); + hdfsSiteProperties.put("dfs.datanode.hostname", HadoopContainer.HOST_NAME); + hdfsSiteProperties.put("dfs.client.socket-timeout", "180000"); + hdfsSiteProperties.put("dfs.datanode.socket.write.timeout", "600000"); + hdfsSiteProperties.put("dfs.replication", "1"); + hdfsSiteProperties.put("dfs.client.read.shortcircuit", "false"); + hdfsSiteProperties.put("dfs.data.transfer.protection", "authentication"); + hdfsSiteProperties.putAll(getHdfsSiteProperties()); - // Build core-site.xml properties - StringBuilder coreSiteProps = new StringBuilder(); - coreSiteProps.append("hadoop.security.authenticationkerberos\\\n"); - coreSiteProps.append("hadoop.security.authorizationtrue\\\n"); - coreSiteProps.append("hadoop.proxyuser.hive.hosts*\\\n"); - coreSiteProps.append("hadoop.proxyuser.hive.groups*\\\n"); - coreSiteProps.append("hadoop.proxyuser.hive.users*\\\n"); - coreSiteProps.append("hadoop.proxyuser.trino.hosts*\\\n"); - coreSiteProps.append("hadoop.proxyuser.trino.groups*\\\n"); - coreSiteProps.append("hadoop.proxyuser.trino.users*"); - // Add any additional core-site properties from subclasses - for (Map.Entry entry : getCoreSiteProperties().entrySet()) { - coreSiteProps.append("\\\n").append(entry.getKey()) - .append("").append(entry.getValue()).append(""); - } + return """ + #!/bin/bash + set -euo pipefail - // Build hdfs-site.xml properties - StringBuilder hdfsSiteProps = new StringBuilder(); - hdfsSiteProps.append("dfs.namenode.kerberos.principal").append(HDFS_PRINCIPAL).append("@").append(realm).append("\\\n"); - hdfsSiteProps.append("dfs.namenode.keytab.file").append(HADOOP_HDFS_KEYTAB).append("\\\n"); - hdfsSiteProps.append("dfs.namenode.kerberos.internal.spnego.principal").append(HTTP_PRINCIPAL).append("@").append(realm).append("\\\n"); - hdfsSiteProps.append("dfs.datanode.kerberos.principal").append(HDFS_PRINCIPAL).append("@").append(realm).append("\\\n"); - hdfsSiteProps.append("dfs.datanode.keytab.file").append(HADOOP_HDFS_KEYTAB).append("\\\n"); - hdfsSiteProps.append("dfs.web.authentication.kerberos.principal").append(HTTP_PRINCIPAL).append("@").append(realm).append("\\\n"); - hdfsSiteProps.append("dfs.web.authentication.kerberos.keytab").append(HADOOP_HTTP_KEYTAB).append("\\\n"); - hdfsSiteProps.append("dfs.block.access.token.enabletrue\\\n"); - hdfsSiteProps.append("dfs.datanode.address0.0.0.0:50010\\\n"); - hdfsSiteProps.append("dfs.datanode.http.address0.0.0.0:50075\\\n"); - hdfsSiteProps.append("dfs.data.transfer.protectionauthentication\\\n"); - hdfsSiteProps.append("dfs.http.policyHTTP_ONLY\\\n"); - hdfsSiteProps.append("ignore.secure.ports.for.testingtrue"); - // Add any additional hdfs-site properties from subclasses - for (Map.Entry entry : getHdfsSiteProperties().entrySet()) { - hdfsSiteProps.append("\\\n").append(entry.getKey()) - .append("").append(entry.getValue()).append(""); + %1$s + %2$s + %3$s + """.formatted( + generateSiteXmlUpdate("/opt/hadoop/etc/hadoop/core-site.xml", getCoreSiteProperties()), + generateSiteXmlUpdate("/opt/hadoop/etc/hadoop/hdfs-site.xml", hdfsSiteProperties), + generateSiteXmlUpdate("/opt/hive/conf/hive-site.xml", getHiveSiteProperties())); + } + + private static String generateSiteXmlUpdate(String path, Map properties) + { + if (properties.isEmpty()) { + return ""; } - // Build hive-site.xml properties. Subclass properties override defaults by key. - Map hiveSitePropertyValues = new LinkedHashMap<>(); - hiveSitePropertyValues.put("hive.metastore.sasl.enabled", "true"); - hiveSitePropertyValues.put("hive.metastore.kerberos.principal", HIVE_PRINCIPAL + "@" + realm); - hiveSitePropertyValues.put("hive.metastore.kerberos.keytab.file", HADOOP_HIVE_KEYTAB); - hiveSitePropertyValues.put("hive.server2.authentication", "KERBEROS"); - hiveSitePropertyValues.put("hive.server2.authentication.kerberos.principal", HIVE_PRINCIPAL + "@" + realm); - hiveSitePropertyValues.put("hive.server2.authentication.kerberos.keytab", HADOOP_HIVE_KEYTAB); - hiveSitePropertyValues.putAll(getHiveSiteProperties()); - - StringBuilder hiveSiteProps = new StringBuilder(); - boolean firstHiveSiteProperty = true; - for (Map.Entry entry : hiveSitePropertyValues.entrySet()) { - if (!firstHiveSiteProperty) { - hiveSiteProps.append("\\\n"); + StringBuilder xml = new StringBuilder(); + for (Map.Entry entry : properties.entrySet()) { + if (!xml.isEmpty()) { + xml.append("\\\n"); } - firstHiveSiteProperty = false; - hiveSiteProps.append("").append(entry.getKey()) + xml.append("").append(entry.getKey()) .append("").append(entry.getValue()).append(""); } return """ - #!/bin/bash - # Don't use set -e to ensure all commands run even if some fail - # set -e - - echo "==========================================" - echo "KERBEROS INIT SCRIPT STARTING" - echo "==========================================" - - REALM="%1$s" - HADOOP_CONF="/opt/hadoop/etc/hadoop" - HIVE_CONF="/opt/hive/conf" - KEYTAB_DIR="%2$s" - - echo "=== Verifying Kerberos files exist ===" - ls -la /etc/krb5.conf - ls -la ${KEYTAB_DIR}/ - - echo "=== Ensuring Kerberos workstation tools are available ===" - if ! command -v kinit >/dev/null 2>&1; then - yum install -y -q krb5-workstation - else - echo "krb5-workstation already present; skipping yum install" - fi - - echo "=== Configuring supervisord child process environment for Kerberos ===" - # Add JAVA_TOOL_OPTIONS to each supervisord program config so JVM processes can find krb5.conf - # The environment= setting must be in each [program:xxx] section, not in [supervisord] - for conf in /etc/supervisord.d/*.conf; do - # Insert environment line after line 1 (after [program:xxx] header) - sed -i '2i environment=JAVA_TOOL_OPTIONS="-Djava.security.krb5.conf=/etc/krb5.conf"' "$conf" - done - - # Make DataNode log to stdout so we can see errors - sed -i 's|stdout_logfile=.*|stdout_logfile=/dev/stdout|' /etc/supervisord.d/hdfs-datanode.conf - sed -i '/stdout_logfile=/a stdout_logfile_maxbytes=0' /etc/supervisord.d/hdfs-datanode.conf - - echo "Modified hdfs-datanode.conf:" - cat /etc/supervisord.d/hdfs-datanode.conf - - - echo "=== Adding Kerberos properties to core-site.xml ===" - sed -i '/<\\/configuration>/i \\ - %3$s' \\ - ${HADOOP_CONF}/core-site.xml - - echo "=== Adding Kerberos properties to hdfs-site.xml ===" - # DataNode must use non-privileged ports in Kerberos mode to avoid requiring JSVC - # SASL data transfer protection enables Kerberos authentication for data transfers - sed -i '/<\\/configuration>/i \\ - %4$s' \\ - ${HADOOP_CONF}/hdfs-site.xml - - echo "=== Adding Kerberos properties to hive-site.xml ===" sed -i '/<\\/configuration>/i \\ - %5$s' \\ - ${HIVE_CONF}/hive-site.xml - - # Keep existing metastore URI property but update host from localhost to Kerberos principal host. - # Duplicate hive.metastore.uris entries are ambiguous; in-place replacement avoids precedence surprises. - sed -i 's|thrift://localhost:9083|thrift://%6$s:%7$s|' ${HIVE_CONF}/hive-site.xml - - echo "=== Verifying keytabs ===" - echo "hdfs.keytab:" - klist -kt ${KEYTAB_DIR}/hdfs.keytab - ls -la ${KEYTAB_DIR}/hdfs.keytab - hexdump -C ${KEYTAB_DIR}/hdfs.keytab | head -5 - echo "hive.keytab:" - klist -kt ${KEYTAB_DIR}/hive.keytab - - echo "=== Testing keytab login ===" - # Try to authenticate with the keytab to verify it works - kinit -kt ${KEYTAB_DIR}/hdfs.keytab hdfs/hadoop-master@${REALM} && echo "kinit successful" || echo "kinit FAILED" - - echo "=== Kerberos configuration complete ===" - echo "==========================================" - echo "KERBEROS INIT SCRIPT FINISHED" - echo "==========================================" - """.formatted( - realm, // %1$s - REALM - HADOOP_KEYTAB_DIR, // %2$s - KEYTAB_DIR - coreSiteProps, // %3$s - core-site.xml properties - hdfsSiteProps, // %4$s - hdfs-site.xml properties - hiveSiteProps, // %5$s - hive-site.xml properties - HadoopContainer.HOST_NAME, // %6$s - Hive Metastore host - HadoopContainer.HIVE_METASTORE_PORT); // %7$s - Hive Metastore port + %s' %s + """.formatted(xml, path); } /** From baba6f2f4041e5d246d71d2ec920e6a90e208ad1 Mon Sep 17 00:00:00 2001 From: Dain Sundstrom Date: Wed, 2 Sep 2026 12:29:27 -0700 Subject: [PATCH 07/18] Use Kerberos image for two Hive clusters --- .../hive/TwoKerberosHivesEnvironment.java | 125 ++++++------------ 1 file changed, 42 insertions(+), 83 deletions(-) diff --git a/testing/trino-product-tests/src/test/java/io/trino/tests/product/hive/TwoKerberosHivesEnvironment.java b/testing/trino-product-tests/src/test/java/io/trino/tests/product/hive/TwoKerberosHivesEnvironment.java index aff0fcfcb81a..002669ac2504 100644 --- a/testing/trino-product-tests/src/test/java/io/trino/tests/product/hive/TwoKerberosHivesEnvironment.java +++ b/testing/trino-product-tests/src/test/java/io/trino/tests/product/hive/TwoKerberosHivesEnvironment.java @@ -73,9 +73,13 @@ *
  • hdfs/hadoop-master@REALM - for Hadoop #1 HDFS services
    • *
  • hive/hadoop-master@REALM - for Hadoop #1 Hive Metastore
    • *
  • HTTP/hadoop-master@REALM - for Hadoop #1 WebHDFS/HTTP SPNEGO
    • + *
  • mapred/hadoop-master@REALM - for Hadoop #1 MapReduce services
    • + *
  • yarn/hadoop-master@REALM - for Hadoop #1 YARN services
    • *
  • hdfs/hadoop-master-2@REALM - for Hadoop #2 HDFS services
    • *
  • hive/hadoop-master-2@REALM - for Hadoop #2 Hive Metastore
    • *
  • HTTP/hadoop-master-2@REALM - for Hadoop #2 WebHDFS/HTTP SPNEGO
    • + *
  • mapred/hadoop-master-2@REALM - for Hadoop #2 MapReduce services
    • + *
  • yarn/hadoop-master-2@REALM - for Hadoop #2 YARN services
    • *
  • trino/trino-master@REALM - for Trino service
    • * */ @@ -94,18 +98,26 @@ public class TwoKerberosHivesEnvironment private static final String HDFS1_PRINCIPAL = "hdfs/" + HADOOP1_HOST; private static final String HIVE1_PRINCIPAL = "hive/" + HADOOP1_HOST; private static final String HTTP1_PRINCIPAL = "HTTP/" + HADOOP1_HOST; + private static final String MAPRED1_PRINCIPAL = "mapred/" + HADOOP1_HOST; + private static final String YARN1_PRINCIPAL = "yarn/" + HADOOP1_HOST; private static final String HDFS2_PRINCIPAL = "hdfs/" + HADOOP2_HOST; private static final String HIVE2_PRINCIPAL = "hive/" + HADOOP2_HOST; private static final String HTTP2_PRINCIPAL = "HTTP/" + HADOOP2_HOST; + private static final String MAPRED2_PRINCIPAL = "mapred/" + HADOOP2_HOST; + private static final String YARN2_PRINCIPAL = "yarn/" + HADOOP2_HOST; private static final String TRINO_PRINCIPAL = "trino/trino-master"; // Keytab paths in KDC container private static final String KDC_HDFS1_KEYTAB_PATH = "/keytabs/hdfs1.keytab"; private static final String KDC_HIVE1_KEYTAB_PATH = "/keytabs/hive1.keytab"; - private static final String KDC_HTTP1_KEYTAB_PATH = "/keytabs/http1.keytab"; + private static final String KDC_HTTP1_KEYTAB_PATH = "/keytabs/HTTP1.keytab"; + private static final String KDC_MAPRED1_KEYTAB_PATH = "/keytabs/mapred1.keytab"; + private static final String KDC_YARN1_KEYTAB_PATH = "/keytabs/yarn1.keytab"; private static final String KDC_HDFS2_KEYTAB_PATH = "/keytabs/hdfs2.keytab"; private static final String KDC_HIVE2_KEYTAB_PATH = "/keytabs/hive2.keytab"; - private static final String KDC_HTTP2_KEYTAB_PATH = "/keytabs/http2.keytab"; + private static final String KDC_HTTP2_KEYTAB_PATH = "/keytabs/HTTP2.keytab"; + private static final String KDC_MAPRED2_KEYTAB_PATH = "/keytabs/mapred2.keytab"; + private static final String KDC_YARN2_KEYTAB_PATH = "/keytabs/yarn2.keytab"; private static final String KDC_TRINO_KEYTAB_PATH = "/keytabs/trino.keytab"; // Path where keytabs are mounted in Hadoop containers @@ -146,10 +158,14 @@ public void start() .withPrincipal(HDFS1_PRINCIPAL, KDC_HDFS1_KEYTAB_PATH) .withPrincipal(HIVE1_PRINCIPAL, KDC_HIVE1_KEYTAB_PATH) .withPrincipal(HTTP1_PRINCIPAL, KDC_HTTP1_KEYTAB_PATH) + .withPrincipal(MAPRED1_PRINCIPAL, KDC_MAPRED1_KEYTAB_PATH) + .withPrincipal(YARN1_PRINCIPAL, KDC_YARN1_KEYTAB_PATH) // Hadoop cluster 2 principals .withPrincipal(HDFS2_PRINCIPAL, KDC_HDFS2_KEYTAB_PATH) .withPrincipal(HIVE2_PRINCIPAL, KDC_HIVE2_KEYTAB_PATH) .withPrincipal(HTTP2_PRINCIPAL, KDC_HTTP2_KEYTAB_PATH) + .withPrincipal(MAPRED2_PRINCIPAL, KDC_MAPRED2_KEYTAB_PATH) + .withPrincipal(YARN2_PRINCIPAL, KDC_YARN2_KEYTAB_PATH) // Trino principal .withPrincipal(TRINO_PRINCIPAL, KDC_TRINO_KEYTAB_PATH); kdc.start(); @@ -195,12 +211,16 @@ private void writeKerberosFiles() // Write cluster 1 keytabs (named as expected by Hadoop) writeKeytab(keytab1Dir, "hdfs.keytab", KDC_HDFS1_KEYTAB_PATH); writeKeytab(keytab1Dir, "hive.keytab", KDC_HIVE1_KEYTAB_PATH); - writeKeytab(keytab1Dir, "http.keytab", KDC_HTTP1_KEYTAB_PATH); + writeKeytab(keytab1Dir, "HTTP.keytab", KDC_HTTP1_KEYTAB_PATH); + writeKeytab(keytab1Dir, "mapred.keytab", KDC_MAPRED1_KEYTAB_PATH); + writeKeytab(keytab1Dir, "yarn.keytab", KDC_YARN1_KEYTAB_PATH); // Write cluster 2 keytabs (named as expected by Hadoop) writeKeytab(keytab2Dir, "hdfs.keytab", KDC_HDFS2_KEYTAB_PATH); writeKeytab(keytab2Dir, "hive.keytab", KDC_HIVE2_KEYTAB_PATH); - writeKeytab(keytab2Dir, "http.keytab", KDC_HTTP2_KEYTAB_PATH); + writeKeytab(keytab2Dir, "HTTP.keytab", KDC_HTTP2_KEYTAB_PATH); + writeKeytab(keytab2Dir, "mapred.keytab", KDC_MAPRED2_KEYTAB_PATH); + writeKeytab(keytab2Dir, "yarn.keytab", KDC_YARN2_KEYTAB_PATH); // Write Trino keytab writeKeytab(trinoKeytabDir, "trino.keytab", KDC_TRINO_KEYTAB_PATH); @@ -236,13 +256,10 @@ private void writeKeytab(Path keytabDir, String fileName, String kdcPath) private HadoopContainer createKerberosHadoopContainer(String hostName, String keytabDirName, String initDirName) { - HadoopContainer container = HadoopContainer.withHostName(hostName) + HadoopContainer container = HadoopContainer.kerberizedWithHostName(hostName) .withNetwork(network) .withNetworkAliases(hostName); - // Set JAVA_TOOL_OPTIONS so all JVM processes can find krb5.conf - container.withEnv("JAVA_TOOL_OPTIONS", "-Djava.security.krb5.conf=/etc/krb5.conf"); - // Bind mount Kerberos files container.withFileSystemBind( tempDir.resolve("krb5.conf").toString(), @@ -265,84 +282,26 @@ private HadoopContainer createKerberosHadoopContainer(String hostName, String ke private String generateKerberosInitScript(String hostName) { - String realm = kdc.getRealm(); - String hdfsPrincipal = "hdfs/" + hostName + "@" + realm; - String hivePrincipal = "hive/" + hostName + "@" + realm; - String httpPrincipal = "HTTP/" + hostName + "@" + realm; - return """ #!/bin/bash - echo "==========================================" - echo "KERBEROS INIT SCRIPT STARTING for %1$s" - echo "==========================================" - - HADOOP_CONF="/opt/hadoop/etc/hadoop" - HIVE_CONF="/opt/hive/conf" - KEYTAB_DIR="%2$s" - - echo "=== Ensuring Kerberos workstation tools are available ===" - if ! command -v kinit >/dev/null 2>&1; then - yum install -y -q krb5-workstation - else - echo "krb5-workstation already present; skipping yum install" - fi - - echo "=== Configuring supervisord for Kerberos ===" - for conf in /etc/supervisord.d/*.conf; do - sed -i '2i environment=JAVA_TOOL_OPTIONS="-Djava.security.krb5.conf=/etc/krb5.conf"' "$conf" - done - - echo "=== Adding Kerberos properties to core-site.xml ===" - sed -i '/<\\/configuration>/i \\ - hadoop.security.authenticationkerberos\\ - hadoop.security.authorizationtrue\\ - hadoop.proxyuser.hive.hosts*\\ - hadoop.proxyuser.hive.groups*\\ - hadoop.proxyuser.hive.users*\\ - hadoop.proxyuser.trino.hosts*\\ - hadoop.proxyuser.trino.groups*\\ - hadoop.proxyuser.trino.users*' \\ - ${HADOOP_CONF}/core-site.xml - - echo "=== Adding Kerberos properties to hdfs-site.xml ===" - sed -i '/<\\/configuration>/i \\ - dfs.namenode.kerberos.principal%3$s\\ - dfs.namenode.keytab.file%2$s/hdfs.keytab\\ - dfs.namenode.kerberos.internal.spnego.principal%4$s\\ - dfs.datanode.kerberos.principal%3$s\\ - dfs.datanode.keytab.file%2$s/hdfs.keytab\\ - dfs.web.authentication.kerberos.principal%4$s\\ - dfs.web.authentication.kerberos.keytab%2$s/http.keytab\\ - dfs.block.access.token.enabletrue\\ - dfs.datanode.address0.0.0.0:50010\\ - dfs.datanode.http.address0.0.0.0:50075\\ - dfs.data.transfer.protectionauthentication\\ - dfs.http.policyHTTP_ONLY\\ - ignore.secure.ports.for.testingtrue' \\ - ${HADOOP_CONF}/hdfs-site.xml - - echo "=== Adding Kerberos properties to hive-site.xml ===" + set -euo pipefail + + sed -i 's/%1$s/%2$s/g' \ + /opt/hadoop/etc/hadoop/*-site.xml \ + /opt/hive/conf/*-site.xml \ + /etc/hadoop-init.d/init-hdfs.sh + sed -i '/<\\/configuration>/i \\ - hive.metastore.sasl.enabledtrue\\ - hive.metastore.kerberos.principal%5$s\\ - hive.metastore.kerberos.keytab.file%2$s/hive.keytab\\ - hive.server2.authenticationKERBEROS\\ - hive.server2.authentication.kerberos.principal%5$s\\ - hive.server2.authentication.kerberos.keytab%2$s/hive.keytab' \\ - ${HIVE_CONF}/hive-site.xml - - echo "=== Testing keytab ===" - kinit -kt ${KEYTAB_DIR}/hdfs.keytab %3$s && echo "kinit successful" || echo "kinit FAILED" - - echo "==========================================" - echo "KERBEROS INIT SCRIPT FINISHED for %1$s" - echo "==========================================" - """.formatted( - hostName, // %1$s - hostname - HADOOP_KEYTAB_DIR, // %2$s - KEYTAB_DIR - hdfsPrincipal, // %3$s - HDFS principal with realm - httpPrincipal, // %4$s - HTTP principal with realm - hivePrincipal); // %5$s - Hive principal with realm + dfs.client.use.datanode.hostnametrue\\ + dfs.datanode.use.datanode.hostnametrue\\ + dfs.datanode.hostname%2$s\\ + dfs.client.socket-timeout180000\\ + dfs.datanode.socket.write.timeout600000\\ + dfs.replication1\\ + dfs.client.read.shortcircuitfalse\\ + dfs.data.transfer.protectionauthentication' \\ + /opt/hadoop/etc/hadoop/hdfs-site.xml + """.formatted(HadoopContainer.HOST_NAME, hostName); } private String getKerberosHdfsClientSiteXml(String hostName) From 83129f579dd499561c4cf04e1ba9f1f3e6f74687 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Fri, 4 Sep 2026 21:37:43 +0900 Subject: [PATCH 08/18] Include BigQuery trace id in debug message --- .../java/io/trino/plugin/bigquery/BigQuerySplit.java | 10 ++++++---- .../io/trino/plugin/bigquery/BigQuerySplitSource.java | 2 +- .../bigquery/BigQueryStorageArrowPageSource.java | 4 ++-- .../plugin/bigquery/BigQueryStorageAvroPageSource.java | 6 ++++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplit.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplit.java index d7979b2d35ea..cf148ba19201 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplit.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplit.java @@ -28,6 +28,7 @@ public record BigQuerySplit( Mode mode, + String traceId, String streamName, String schemaString, List columns, @@ -44,6 +45,7 @@ public record BigQuerySplit( public BigQuerySplit { requireNonNull(mode, "mode is null"); + requireNonNull(traceId, "traceId is null"); requireNonNull(streamName, "streamName cannot be null"); requireNonNull(schemaString, "schemaString cannot be null"); columns = ImmutableList.copyOf(requireNonNull(columns, "columns cannot be null")); @@ -51,19 +53,19 @@ public record BigQuerySplit( requireNonNull(dataSize, "dataSize is null"); } - static BigQuerySplit forStream(String streamName, String schemaString, List columns, OptionalInt dataSize) + static BigQuerySplit forStream(String traceId, String streamName, String schemaString, List columns, OptionalInt dataSize) { - return new BigQuerySplit(STORAGE, streamName, schemaString, columns, NO_ROWS_TO_GENERATE, Optional.empty(), dataSize); + return new BigQuerySplit(STORAGE, traceId, streamName, schemaString, columns, NO_ROWS_TO_GENERATE, Optional.empty(), dataSize); } static BigQuerySplit forViewStream(List columns, Optional filter) { - return new BigQuerySplit(QUERY, "", "", columns, NO_ROWS_TO_GENERATE, filter, OptionalInt.empty()); + return new BigQuerySplit(QUERY, "", "", "", columns, NO_ROWS_TO_GENERATE, filter, OptionalInt.empty()); } static BigQuerySplit emptyProjection(long numberOfRows) { - return new BigQuerySplit(STORAGE, "", "", ImmutableList.of(), numberOfRows, Optional.empty(), OptionalInt.of(0)); + return new BigQuerySplit(STORAGE, "", "", "", ImmutableList.of(), numberOfRows, Optional.empty(), OptionalInt.of(0)); } @Override diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplitSource.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplitSource.java index 674146957fff..cae750f8da74 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplitSource.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQuerySplitSource.java @@ -212,7 +212,7 @@ private List readFromBigQuery( String schemaString = getSchemaAsString(readSession); return readSession.getStreamsList().stream() - .map(stream -> BigQuerySplit.forStream(stream.getName(), schemaString, columns, OptionalInt.of(stream.getSerializedSize()))) + .map(stream -> BigQuerySplit.forStream(readSession.getTraceId(), stream.getName(), schemaString, columns, OptionalInt.of(stream.getSerializedSize()))) .collect(toImmutableList()); } diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageArrowPageSource.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageArrowPageSource.java index 2e646cd44536..78357551d3de 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageArrowPageSource.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageArrowPageSource.java @@ -75,7 +75,7 @@ public BigQueryStorageArrowPageSource( this.split = requireNonNull(split, "split is null"); requireNonNull(columns, "columns is null"); Schema schema = deserializeSchema(split.schemaString()); - log.debug("Starting to read from %s", split.streamName()); + log.debug("Trace id: %s, Stream: %s, Starting to read", split.traceId(), split.streamName()); responses = new ReadRowsHelper(bigQueryReadClient, split.streamName(), maxReadRowsRetries).readRows(); nextResponse = CompletableFuture.supplyAsync(this::getResponse, executor); this.bigQueryArrowToPageConverter = new BigQueryArrowToPageConverter(typeManager, schema, columns); @@ -165,7 +165,7 @@ private ArrowRecordBatch deserializeResponse(BufferAllocator allocator, ReadRows { int serializedSize = response.getArrowRecordBatch().getSerializedSize(); long totalReadSize = readBytes.addAndGet(serializedSize); - log.debug("Read %d bytes (total %d) from %s", serializedSize, totalReadSize, split.streamName()); + log.debug("Trace id: %s, Stream: %s, Read %d bytes (total %d)", split.traceId(), split.streamName(), serializedSize, totalReadSize); try { return MessageSerializer.deserializeRecordBatch(readChannelForByteString(response.getArrowRecordBatch().getSerializedRecordBatch()), allocator); diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageAvroPageSource.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageAvroPageSource.java index 5e0176d65da0..089a4a67a63d 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageAvroPageSource.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageAvroPageSource.java @@ -92,6 +92,7 @@ public class BigQueryStorageAvroPageSource private final BigQueryReadClient bigQueryReadClient; private final ExecutorService executor; private final BigQueryTypeManager typeManager; + private final String traceId; private final String streamName; private final Schema avroSchema; private final List columns; @@ -115,6 +116,7 @@ public BigQueryStorageAvroPageSource( this.executor = requireNonNull(executor, "executor is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); requireNonNull(split, "split is null"); + this.traceId = split.traceId(); this.streamName = split.streamName(); this.avroSchema = parseSchema(split.schemaString()); this.columns = requireNonNull(columns, "columns is null"); @@ -122,7 +124,7 @@ public BigQueryStorageAvroPageSource( .map(BigQueryColumnHandle::trinoType) .collect(toImmutableList())); - log.debug("Starting to read from %s", streamName); + log.debug("Trace id: %s, Stream: %s, Starting to read", traceId, streamName); responses = new ReadRowsHelper(bigQueryReadClient, streamName, maxReadRowsRetries).readRows(); nextResponse = CompletableFuture.supplyAsync(this::getResponse, executor); } @@ -359,7 +361,7 @@ Iterable parse(ReadRowsResponse response) { byte[] buffer = response.getAvroRows().getSerializedBinaryRows().toByteArray(); readBytes.addAndGet(buffer.length); - log.debug("Read %d bytes (total %d) from %s", buffer.length, readBytes.get(), streamName); + log.debug("Trace id: %s, Stream: %s, Read %d bytes (total %d)", traceId, streamName, buffer.length, readBytes.get()); return () -> new AvroBinaryIterator(avroSchema, buffer); } From b759085d48444880ae50ce89b794ea77cd1f95ec Mon Sep 17 00:00:00 2001 From: Xi Chen Date: Mon, 7 Sep 2026 10:09:00 +0800 Subject: [PATCH 09/18] Enable Iceberg Parquet page skipping with column indexes Skip unread Parquet pages when column indexes are present. Pass the predicate into ParquetReader unconditionally. DELETE, UPDATE, MERGE, and $row_id skip pages and still match skip-off results. Tests extend Hive BaseTestParquetPageSkipping and append Hive fixtures as Iceberg DataFiles. --- .../sphinx/object-storage/file-formats.md | 3 +- .../TestDeltaLakeParquetPageSkipping.java | 23 -- .../hive/BaseTestParquetPageSkipping.java | 21 +- .../iceberg/IcebergPageSourceProvider.java | 6 +- .../iceberg/IcebergSessionProperties.java | 11 + .../TestIcebergParquetPageSkipping.java | 313 ++++++++++++++++++ 6 files changed, 343 insertions(+), 34 deletions(-) create mode 100644 plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetPageSkipping.java diff --git a/docs/src/main/sphinx/object-storage/file-formats.md b/docs/src/main/sphinx/object-storage/file-formats.md index 1490e61bdc09..2bdb9e1e1435 100644 --- a/docs/src/main/sphinx/object-storage/file-formats.md +++ b/docs/src/main/sphinx/object-storage/file-formats.md @@ -94,8 +94,7 @@ with Parquet files performed by supported object storage connectors: - `true` * - `parquet.use-column-index` - Skip reading Parquet pages by using Parquet column indices. The equivalent - catalog session property is `parquet_use_column_index`. Only supported by - the Delta Lake and Hive connectors. + catalog session property is `parquet_use_column_index`. - `true` * - `parquet.ignore-statistics` - Ignore statistics from Parquet to allow querying files with corrupted or diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeParquetPageSkipping.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeParquetPageSkipping.java index 2a360540d2af..7eba5eec402b 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeParquetPageSkipping.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeParquetPageSkipping.java @@ -15,21 +15,16 @@ import com.google.common.io.Resources; import io.trino.plugin.hive.BaseTestParquetPageSkipping; -import io.trino.spi.metrics.Count; import io.trino.testing.QueryRunner; -import io.trino.testing.QueryRunner.MaterializedResultWithPlan; -import org.intellij.lang.annotations.Language; import org.junit.jupiter.api.Test; import java.io.IOException; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Objects; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; -import static io.trino.parquet.reader.ParquetReader.COLUMN_INDEX_ROWS_FILTERED; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; @@ -179,22 +174,4 @@ private String createTableWithDataFile(String tableNamePrefix, String columnsDef Files.writeString(tableLocation.resolve("_delta_log").resolve("00000000000000000001.json"), addAction + "\n"); return tableName; } - - private void assertUpdateWithPageSkipping(@Language("SQL") String sql, long expectedUpdateCount) - { - MaterializedResultWithPlan result = getDistributedQueryRunner().executeWithPlan(getSession(), sql); - assertThat(result.result().getUpdateCount()).hasValue(expectedUpdateCount); - long rowsFilteredByColumnIndex = getDistributedQueryRunner().getCoordinator() - .getQueryManager() - .getFullQueryInfo(result.queryId()) - .getQueryStats() - .getOperatorSummaries() - .stream() - .filter(summary -> summary.getOperatorType().startsWith("TableScan") || summary.getOperatorType().startsWith("Scan")) - .map(summary -> summary.getConnectorMetrics().getMetrics().get(COLUMN_INDEX_ROWS_FILTERED)) - .filter(Objects::nonNull) - .mapToLong(metric -> ((Count) metric).getTotal()) - .sum(); - assertThat(rowsFilteredByColumnIndex).isGreaterThan(0); - } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestParquetPageSkipping.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestParquetPageSkipping.java index 6391d7b51886..bc822990e2ae 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestParquetPageSkipping.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestParquetPageSkipping.java @@ -169,12 +169,7 @@ protected void verifyFilteringWithColumnIndex(@Language("SQL") String query) assertThat(queryStatsWithColumnIndex.getPhysicalInputPositions()).isGreaterThan(0); assertThat(queryStatsWithColumnIndex.getPhysicalInputPositions()) .isLessThan(queryStatsWithoutColumnIndex.getPhysicalInputPositions()); - Map> metricsWithColumnIndex = getScanOperatorStats(resultWithColumnIndex.queryId()) - .getConnectorMetrics() - .getMetrics(); - assertThat(metricsWithColumnIndex).containsKey(COLUMN_INDEX_ROWS_FILTERED); - assertThat(((Count) metricsWithColumnIndex.get(COLUMN_INDEX_ROWS_FILTERED)).getTotal()) - .isGreaterThan(0); + assertThat(getColumnIndexRowsFiltered(resultWithColumnIndex.queryId())).isGreaterThan(0); assertEqualsIgnoreOrder(resultWithColumnIndex.result(), resultWithoutColumnIndex.result()); } @@ -215,6 +210,13 @@ protected Session noParquetColumnIndexFiltering(Session session) .build(); } + protected void assertUpdateWithPageSkipping(@Language("SQL") String sql, long expectedUpdateCount) + { + MaterializedResultWithPlan result = getDistributedQueryRunner().executeWithPlan(getSession(), sql); + assertThat(result.result().getUpdateCount()).hasValue(expectedUpdateCount); + assertThat(getColumnIndexRowsFiltered(result.queryId())).isGreaterThan(0); + } + protected static String tableName(String tableNamePrefix) { return tableNamePrefix + "_" + randomNameSuffix(); @@ -236,4 +238,11 @@ private OperatorStats getScanOperatorStats(QueryId queryId) .filter(summary -> summary.getOperatorType().startsWith("TableScan") || summary.getOperatorType().startsWith("Scan")) .collect(onlyElement()); } + + private long getColumnIndexRowsFiltered(QueryId queryId) + { + Map> metrics = getScanOperatorStats(queryId).getConnectorMetrics().getMetrics(); + assertThat(metrics).containsKey(COLUMN_INDEX_ROWS_FILTERED); + return ((Count) metrics.get(COLUMN_INDEX_ROWS_FILTERED)).getTotal(); + } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java index 6ab21cd058ac..f00cde21ca35 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java @@ -199,6 +199,7 @@ import static io.trino.plugin.iceberg.IcebergSessionProperties.isOrcBloomFiltersEnabled; import static io.trino.plugin.iceberg.IcebergSessionProperties.isOrcNestedLazy; import static io.trino.plugin.iceberg.IcebergSessionProperties.isParquetIgnoreStatistics; +import static io.trino.plugin.iceberg.IcebergSessionProperties.isParquetUseColumnIndex; import static io.trino.plugin.iceberg.IcebergSessionProperties.isParquetVectorizedDecodingEnabled; import static io.trino.plugin.iceberg.IcebergSessionProperties.isUseFileSizeFromMetadata; import static io.trino.plugin.iceberg.IcebergSessionProperties.useParquetBloomFilter; @@ -681,8 +682,7 @@ private ReaderPageSourceWithRowPositions createDataPageSource( .withSmallFileThreshold(getParquetSmallFileThreshold(session)) .withIgnoreStatistics(isParquetIgnoreStatistics(session)) .withBloomFilter(useParquetBloomFilter(session)) - // TODO https://github.com/trinodb/trino/issues/11000 - .withUseColumnIndex(false) + .withUseColumnIndex(isParquetUseColumnIndex(session)) .withVectorizedDecodingEnabled(isParquetVectorizedDecodingEnabled(session)) .build(), predicate, @@ -1286,7 +1286,7 @@ else if (column.isBaseColumn()) { memoryContext, options, exception -> handleException(dataSourceId, exception), - Optional.empty(), + Optional.of(parquetPredicate), Optional.empty(), parquetMetadata.getDecryptionContext()); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java index a043d3516ebf..3719fe1cdd6a 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSessionProperties.java @@ -83,6 +83,7 @@ public final class IcebergSessionProperties private static final String ORC_WRITER_MAX_DICTIONARY_MEMORY = "orc_writer_max_dictionary_memory"; private static final String PARQUET_MAX_READ_BLOCK_SIZE = "parquet_max_read_block_size"; private static final String PARQUET_USE_BLOOM_FILTER = "parquet_use_bloom_filter"; + private static final String PARQUET_USE_COLUMN_INDEX = "parquet_use_column_index"; private static final String PARQUET_MAX_READ_BLOCK_ROW_COUNT = "parquet_max_read_block_row_count"; private static final String PARQUET_SMALL_FILE_THRESHOLD = "parquet_small_file_threshold"; private static final String PARQUET_IGNORE_STATISTICS = "parquet_ignore_statistics"; @@ -231,6 +232,11 @@ public IcebergSessionProperties( "Use Parquet Bloom filters", parquetReaderConfig.isUseBloomFilter(), false)) + .add(booleanProperty( + PARQUET_USE_COLUMN_INDEX, + "Use Parquet column index", + parquetReaderConfig.isUseColumnIndex(), + false)) .add(integerProperty( PARQUET_MAX_READ_BLOCK_ROW_COUNT, "Parquet: Maximum number of rows read in a batch", @@ -569,6 +575,11 @@ public static boolean useParquetBloomFilter(ConnectorSession session) return session.getProperty(PARQUET_USE_BLOOM_FILTER, Boolean.class); } + public static boolean isParquetUseColumnIndex(ConnectorSession session) + { + return session.getProperty(PARQUET_USE_COLUMN_INDEX, Boolean.class); + } + public static Duration getDynamicFilteringWaitTimeout(ConnectorSession session) { return session.getProperty(DYNAMIC_FILTERING_WAIT_TIMEOUT, Duration.class); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetPageSkipping.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetPageSkipping.java new file mode 100644 index 000000000000..becefba83409 --- /dev/null +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetPageSkipping.java @@ -0,0 +1,313 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg; + +import com.google.common.io.Resources; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; +import io.trino.metastore.HiveMetastore; +import io.trino.parquet.metadata.BlockMetadata; +import io.trino.parquet.metadata.ParquetMetadata; +import io.trino.plugin.hive.BaseTestParquetPageSkipping; +import io.trino.testing.QueryRunner; +import org.apache.iceberg.AppendFiles; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Schema; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.data.parquet.GenericParquetWriter; +import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.mapping.MappingUtil; +import org.apache.iceberg.parquet.Parquet; +import org.intellij.lang.annotations.Language; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +import static com.google.common.io.Resources.getResource; +import static io.trino.plugin.iceberg.IcebergQueryRunner.ICEBERG_CATALOG; +import static io.trino.plugin.iceberg.IcebergTestUtils.SESSION; +import static io.trino.plugin.iceberg.IcebergTestUtils.getFileSystemFactory; +import static io.trino.plugin.iceberg.IcebergTestUtils.getHiveMetastore; +import static io.trino.plugin.iceberg.IcebergTestUtils.getParquetFileMetadata; +import static io.trino.testing.TestingNames.randomNameSuffix; +import static java.lang.String.format; +import static org.apache.iceberg.TableProperties.DEFAULT_NAME_MAPPING; +import static org.apache.iceberg.TableProperties.PARQUET_PAGE_ROW_LIMIT; +import static org.apache.iceberg.TableProperties.PARQUET_PAGE_SIZE_BYTES; +import static org.apache.iceberg.TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES; +import static org.apache.iceberg.mapping.NameMappingParser.toJson; +import static org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_2_0; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestIcebergParquetPageSkipping + extends BaseTestParquetPageSkipping +{ + private TrinoFileSystem fileSystem; + private HiveMetastore metastore; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return IcebergQueryRunner.builder() + .addIcebergProperty("iceberg.file-format", "PARQUET") + .addIcebergProperty("parquet.use-column-index", "true") + .addIcebergProperty("parquet.max-buffer-size", "1MB") + .build(); + } + + @BeforeAll + public void setUp() + { + fileSystem = getFileSystemFactory(getQueryRunner()).create(SESSION); + metastore = getHiveMetastore(getQueryRunner()); + } + + @Override + protected String createTableWithDataFile(String tableNamePrefix, String columnsDefinition, String resourceFileName) + throws Exception + { + String tableName = tableName(tableNamePrefix); + assertUpdate(format("CREATE TABLE %s %s WITH (format = 'PARQUET')", tableName, columnsDefinition)); + BaseTable table = loadTable(tableName); + table.updateProperties() + .set(DEFAULT_NAME_MAPPING, toJson(MappingUtil.create(table.schema()))) + .commit(); + appendIndexedFile(tableName, resourceFileName, Optional.empty()); + return tableName; + } + + @Override + protected String timestampMillisType() + { + return "timestamp(6)"; + } + + @Test + public void testPartitionEvolutionDoesNotReturnEmpty() + throws Exception + { + String tableName = createTableWithDataFile( + "test_partition_evolution", + """ + ( + orderkey bigint, + custkey bigint, + orderstatus varchar, + totalprice double, + orderdate date, + orderpriority varchar, + clerk varchar, + shippriority integer, + comment varchar, + rvalues array(double)) + """, + "parquet_page_skipping/orders_sorted_by_totalprice/data.parquet"); + assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES partitioning = ARRAY['orderstatus']"); + appendIndexedFile( + tableName, + "parquet_page_skipping/orders_sorted_by_totalprice/data.parquet", + Optional.of("O")); + @Language("SQL") String query = "SELECT orderkey FROM " + tableName + + " WHERE orderstatus = 'O' AND totalprice BETWEEN 100000 AND 131280"; + assertThat(assertColumnIndexResults(query)).isGreaterThan(0); + assertUpdate("DROP TABLE " + tableName); + } + + @Test + public void testPositionDeletesWithPageSkipping() + throws Exception + { + testDeletesWithPageSkipping(2); + } + + @Test + public void testDeletionVectorsWithPageSkipping() + throws Exception + { + testDeletesWithPageSkipping(3); + } + + private void testDeletesWithPageSkipping(int formatVersion) + throws Exception + { + String tableName = createParquetV2IndexedTable(formatVersion); + @Language("SQL") String neighbors = "SELECT id, payload FROM " + tableName + " WHERE id BETWEEN 8 AND 12 ORDER BY id"; + assertQuery(neighbors, "VALUES (8, 'row-8'), (9, 'row-9'), (10, 'row-10'), (11, 'row-11'), (12, 'row-12')"); + + assertUpdateWithPageSkipping("DELETE FROM " + tableName + " WHERE id = 10", 1); + assertQuery(neighbors, "VALUES (8, 'row-8'), (9, 'row-9'), (11, 'row-11'), (12, 'row-12')"); + assertQueryReturnsEmptyResult("SELECT id FROM " + tableName + " WHERE id = 10"); + verifyFilteringWithColumnIndex("SELECT id, payload FROM " + tableName + " WHERE id BETWEEN 8 AND 12"); + assertUpdate("DROP TABLE " + tableName); + } + + @Test + public void testParquetV2PageSkipping() + throws Exception + { + String tableName = createParquetV2IndexedTable(2); + assertParquetV2Pages(tableName); + verifyFilteringWithColumnIndex("SELECT * FROM " + tableName + " WHERE id = 10"); + assertUpdate("DROP TABLE " + tableName); + } + + @Test + public void testRowIdWithPageSkipping() + throws Exception + { + String tableName = createParquetV2IndexedTable(3, 2); + assertUpdateWithPageSkipping("DELETE FROM " + tableName + " WHERE id = 1500", 2); + assertQuery( + "SELECT id, \"$row_id\" FROM " + tableName + " WHERE id BETWEEN 1499 AND 1501", + "VALUES (1499, 1499), (1501, 1501), (1499, 3499), (1501, 3501)"); + assertUpdate("DROP TABLE " + tableName); + } + + @Test + public void testUpdateWithPageSkipping() + throws Exception + { + testUpdateWithPageSkipping(2); + testUpdateWithPageSkipping(3); + } + + private void testUpdateWithPageSkipping(int formatVersion) + throws Exception + { + String tableName = createParquetV2IndexedTable(formatVersion); + assertUpdateWithPageSkipping("UPDATE " + tableName + " SET payload = 'updated' WHERE id = 10", 1); + assertQuery( + "SELECT id, payload FROM " + tableName + " WHERE id BETWEEN 8 AND 12 ORDER BY id", + "VALUES (8, 'row-8'), (9, 'row-9'), (10, 'updated'), (11, 'row-11'), (12, 'row-12')"); + verifyFilteringWithColumnIndex("SELECT id, payload FROM " + tableName + " WHERE id BETWEEN 8 AND 12"); + assertUpdate("DROP TABLE " + tableName); + } + + @Test + public void testMergeWithPageSkipping() + throws Exception + { + testMergeWithPageSkipping(2); + testMergeWithPageSkipping(3); + } + + private void testMergeWithPageSkipping(int formatVersion) + throws Exception + { + String tableName = createParquetV2IndexedTable(formatVersion); + assertUpdateWithPageSkipping( + "MERGE INTO " + tableName + " t USING (VALUES BIGINT '10') s(id) ON t.id = s.id " + + "WHEN MATCHED THEN UPDATE SET payload = 'updated'", + 1); + assertQuery( + "SELECT id, payload FROM " + tableName + " WHERE id BETWEEN 8 AND 12 ORDER BY id", + "VALUES (8, 'row-8'), (9, 'row-9'), (10, 'updated'), (11, 'row-11'), (12, 'row-12')"); + assertUpdate("DROP TABLE " + tableName); + } + + private String createParquetV2IndexedTable(int formatVersion) + throws Exception + { + return createParquetV2IndexedTable(formatVersion, 1); + } + + private String createParquetV2IndexedTable(int formatVersion, int fileCopies) + throws Exception + { + String tableName = "test_iceberg_page_skipping_v2_" + randomNameSuffix(); + assertUpdate("CREATE TABLE " + tableName + + " (id bigint, payload varchar) WITH (format = 'PARQUET', format_version = " + formatVersion + ")"); + BaseTable table = loadTable(tableName); + Schema schema = table.schema(); + AppendFiles append = table.newAppend(); + for (int copy = 0; copy < fileCopies; copy++) { + String dataPath = table.location() + "/data/v2-indexed-" + randomNameSuffix() + ".parquet"; + FileAppender writer = Parquet.write(table.io().newOutputFile(dataPath)) + .schema(schema) + .createWriterFunc(GenericParquetWriter::create) + .writerVersion(PARQUET_2_0) + .set(PARQUET_ROW_GROUP_SIZE_BYTES, "4096") + .set(PARQUET_PAGE_ROW_LIMIT, "32") + .set(PARQUET_PAGE_SIZE_BYTES, "256") + .build(); + try { + Record record = GenericRecord.create(schema); + for (long id = 0; id < 2000; id++) { + record.setField("id", id); + record.setField("payload", "row-" + id); + writer.add(record); + } + } + finally { + writer.close(); + } + append.appendFile(DataFiles.builder(table.spec()) + .withPath(dataPath) + .withFormat(FileFormat.PARQUET) + .withFileSizeInBytes(writer.length()) + .withMetrics(writer.metrics()) + .build()); + } + append.commit(); + return tableName; + } + + private BaseTable loadTable(String tableName) + { + return IcebergTestUtils.loadTable(tableName, metastore, getFileSystemFactory(getQueryRunner()), ICEBERG_CATALOG, "tpch"); + } + + private void assertParquetV2Pages(String tableName) + throws Exception + { + String filePath = (String) computeScalar(format("SELECT file_path FROM \"%s$files\"", tableName)); + ParquetMetadata parquetMetadata = getParquetFileMetadata(fileSystem.newInputFile(Location.of(filePath))); + assertThat(parquetMetadata.getBlocks()).isNotEmpty(); + boolean usesV2Pages = parquetMetadata.getBlocks().stream() + .flatMap(block -> block.columns().stream()) + .anyMatch(column -> column.getEncodingStats() != null && column.getEncodingStats().usesV2Pages()); + assertThat(usesV2Pages).isTrue(); + boolean hasColumnIndex = parquetMetadata.getBlocks().stream() + .flatMap(block -> block.columns().stream()) + .anyMatch(column -> column.getColumnIndexReference() != null); + assertThat(hasColumnIndex).isTrue(); + } + + private void appendIndexedFile(String tableName, String resourceName, Optional orderstatus) + throws Exception + { + BaseTable table = loadTable(tableName); + String dataPath = table.location() + "/data/" + randomNameSuffix() + ".parquet"; + byte[] parquetFileData = Resources.toByteArray(getResource(resourceName)); + fileSystem.newOutputFile(Location.of(dataPath)).createOrOverwrite(parquetFileData); + ParquetMetadata parquetMetadata = getParquetFileMetadata(fileSystem.newInputFile(Location.of(dataPath))); + long recordCount = parquetMetadata.getBlocks().stream() + .mapToLong(BlockMetadata::rowCount) + .sum(); + DataFiles.Builder builder = DataFiles.builder(table.spec()) + .withPath(dataPath) + .withFormat(FileFormat.PARQUET) + .withFileSizeInBytes(parquetFileData.length) + .withRecordCount(recordCount); + orderstatus.ifPresent(value -> builder.withPartition(new PartitionData(new Object[] {value}))); + table.newAppend() + .appendFile(builder.build()) + .commit(); + } +} From 887bc6f524fad29b6f9ae8cca5ff96fa4e1ef06b Mon Sep 17 00:00:00 2001 From: chenjian2664 Date: Mon, 7 Sep 2026 11:37:02 +0800 Subject: [PATCH 10/18] Use getConnectorService in Iceberg REST catalog smoke tests --- .../rest/TestIcebergBigLakeMetastoreConnectorSmokeTest.java | 4 ++-- .../rest/TestIcebergPolarisCatalogConnectorSmokeTest.java | 4 ++-- .../catalog/rest/TestIcebergS3TablesConnectorSmokeTest.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergBigLakeMetastoreConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergBigLakeMetastoreConnectorSmokeTest.java index 4e3bc5af673e..53b78043076e 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergBigLakeMetastoreConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergBigLakeMetastoreConnectorSmokeTest.java @@ -20,7 +20,6 @@ import io.trino.filesystem.Location; import io.trino.plugin.iceberg.BaseIcebergConnectorSmokeTest; import io.trino.plugin.iceberg.IcebergConfig; -import io.trino.plugin.iceberg.IcebergConnector; import io.trino.plugin.iceberg.IcebergQueryRunner; import io.trino.plugin.iceberg.SchemaInitializer; import io.trino.plugin.iceberg.catalog.TrinoCatalog; @@ -37,6 +36,7 @@ import java.io.UncheckedIOException; import java.util.Base64; +import static io.trino.plugin.iceberg.IcebergTestUtils.getConnectorService; import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; @@ -117,7 +117,7 @@ void cleanup() @Override protected String getMetadataLocation(String tableName) { - TrinoCatalogFactory catalogFactory = ((IcebergConnector) getQueryRunner().getCoordinator().getConnector("iceberg")).getInjector().getInstance(TrinoCatalogFactory.class); + TrinoCatalogFactory catalogFactory = getConnectorService(getQueryRunner(), TrinoCatalogFactory.class); TrinoCatalog trinoCatalog = catalogFactory.create(getSession().getIdentity().toConnectorIdentity()); BaseTable table = trinoCatalog.loadTable(getSession().toConnectorSession(), new SchemaTableName(getSession().getSchema().orElseThrow(), tableName)); return table.operations().current().metadataFileLocation(); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergPolarisCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergPolarisCatalogConnectorSmokeTest.java index 2c2799633ff0..30d19ee565d6 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergPolarisCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergPolarisCatalogConnectorSmokeTest.java @@ -20,7 +20,6 @@ import io.trino.filesystem.s3.S3FileSystemStats; import io.trino.plugin.iceberg.BaseIcebergConnectorSmokeTest; import io.trino.plugin.iceberg.IcebergConfig; -import io.trino.plugin.iceberg.IcebergConnector; import io.trino.plugin.iceberg.IcebergQueryRunner; import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; @@ -36,6 +35,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import static io.trino.plugin.iceberg.IcebergTestUtils.getConnectorService; import static io.trino.testing.TestingConnectorSession.SESSION; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.testing.containers.Minio.MINIO_REGION; @@ -134,7 +134,7 @@ protected String getTableLocation(String tableName) private BaseTable loadTable(String tableName) { - TrinoCatalogFactory catalogFactory = ((IcebergConnector) getQueryRunner().getCoordinator().getConnector("iceberg")).getInjector().getInstance(TrinoCatalogFactory.class); + TrinoCatalogFactory catalogFactory = getConnectorService(getQueryRunner(), TrinoCatalogFactory.class); TrinoCatalog trinoCatalog = catalogFactory.create(getSession().getIdentity().toConnectorIdentity()); return trinoCatalog.loadTable(getSession().toConnectorSession(), new SchemaTableName(getSession().getSchema().orElseThrow(), tableName)); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergS3TablesConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergS3TablesConnectorSmokeTest.java index d7437a5485a5..0d9ecc8649ca 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergS3TablesConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergS3TablesConnectorSmokeTest.java @@ -15,7 +15,6 @@ import io.trino.plugin.iceberg.BaseIcebergConnectorSmokeTest; import io.trino.plugin.iceberg.IcebergConfig; -import io.trino.plugin.iceberg.IcebergConnector; import io.trino.plugin.iceberg.IcebergQueryRunner; import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; @@ -27,6 +26,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import static io.trino.plugin.iceberg.IcebergTestUtils.getConnectorService; import static io.trino.testing.SystemEnvironmentUtils.requireEnv; import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; @@ -85,7 +85,7 @@ protected QueryRunner createQueryRunner() @Override protected String getMetadataLocation(String tableName) { - TrinoCatalogFactory catalogFactory = ((IcebergConnector) getQueryRunner().getCoordinator().getConnector("iceberg")).getInjector().getInstance(TrinoCatalogFactory.class); + TrinoCatalogFactory catalogFactory = getConnectorService(getQueryRunner(), TrinoCatalogFactory.class); TrinoCatalog trinoCatalog = catalogFactory.create(getSession().getIdentity().toConnectorIdentity()); BaseTable table = trinoCatalog.loadTable(getSession().toConnectorSession(), new SchemaTableName(getSession().getSchema().orElseThrow(), tableName)); return table.operations().current().metadataFileLocation(); From a835b60734086acbddf756afeba33595820fe16f Mon Sep 17 00:00:00 2001 From: Raunaq Morarka Date: Sat, 5 Sep 2026 17:08:54 +0530 Subject: [PATCH 11/18] Account TopN pages by retained size Pages were charged at getSizeInBytes, which excludes the unused capacity of the backing arrays left by block builders. Pages with excess retained bytes are now compacted once. --- .../operator/RowReferencePageManager.java | 27 ++++- .../operator/TestRowReferencePageManager.java | 108 ++++++++++++++++++ 2 files changed, 133 insertions(+), 2 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/operator/RowReferencePageManager.java b/core/trino-main/src/main/java/io/trino/operator/RowReferencePageManager.java index c82f72953ae5..e98286493fab 100644 --- a/core/trino-main/src/main/java/io/trino/operator/RowReferencePageManager.java +++ b/core/trino-main/src/main/java/io/trino/operator/RowReferencePageManager.java @@ -233,12 +233,17 @@ public void close() private final class PageAccounting { private static final int COMPACTION_MIN_FILL_MULTIPLIER = 2; + // Copy a page once when its backing arrays retain over 12.5% and at least 4 KB more than the data they hold; + // block builders size their arrays from the previous page, so retained pages commonly carry unused capacity + private static final int COMPACTION_MAX_SLACK_DIVISOR = 8; + private static final long COMPACTION_MIN_SLACK_BYTES = 4 * 1024; private final int pageId; private Page page; private long[] rowIds; // Start off locked to give the caller time to declare which rows to reference private boolean lockedPage = true; + private boolean compacted; private int activePositions; public PageAccounting(int pageId, Page page) @@ -310,7 +315,20 @@ public boolean isPruneEligible() public boolean isCompactionEligible() { // Compaction is only allowed if the page is unlocked - return !lockedPage && activePositions * COMPACTION_MIN_FILL_MULTIPLIER < page.getPositionCount(); + if (lockedPage) { + return false; + } + return activePositions * COMPACTION_MIN_FILL_MULTIPLIER < page.getPositionCount() || hasExcessRetainedBytes(); + } + + private boolean hasExcessRetainedBytes() + { + if (compacted) { + return false; + } + long sizeInBytes = page.getSizeInBytes(); + long slackBytes = page.getRetainedSizeInBytes() - sizeInBytes; + return slackBytes > Math.max(sizeInBytes / COMPACTION_MAX_SLACK_DIVISOR, COMPACTION_MIN_SLACK_BYTES); } public void compact() @@ -318,6 +336,10 @@ public void compact() checkState(!lockedPage, "Should not attempt compaction when page is locked"); if (activePositions == page.getPositionCount()) { + if (hasExcessRetainedBytes()) { + page.compact(); + compacted = true; + } return; } @@ -338,11 +360,12 @@ public void compact() // Compact page page = page.copyPositions(positionsToKeep, 0, positionsToKeep.length); rowIds = newRowIds; + compacted = true; } public long sizeOf() { - return PAGE_ACCOUNTING_INSTANCE_SIZE + page.getSizeInBytes() + SizeOf.sizeOf(rowIds); + return PAGE_ACCOUNTING_INSTANCE_SIZE + page.getRetainedSizeInBytes() + SizeOf.sizeOf(rowIds); } } diff --git a/core/trino-main/src/test/java/io/trino/operator/TestRowReferencePageManager.java b/core/trino-main/src/test/java/io/trino/operator/TestRowReferencePageManager.java index a78d1ab72b32..1c0355c90c8b 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestRowReferencePageManager.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestRowReferencePageManager.java @@ -16,12 +16,14 @@ import io.trino.spi.Page; import io.trino.spi.block.Block; import io.trino.spi.block.BlockBuilder; +import io.trino.spi.block.DictionaryBlock; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.List; import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.VarcharType.VARCHAR; import static java.lang.Math.toIntExact; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -304,6 +306,112 @@ public void testRowIdRecycling() } } + @Test + public void testRetainedSizeAccounting() + { + RowReferencePageManager pageManager = new RowReferencePageManager(); + + BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 2, 1_000_000); + VARCHAR.writeString(blockBuilder, "a"); + VARCHAR.writeString(blockBuilder, "b"); + Page page = new Page(blockBuilder.build()); + long retainedBefore = page.getRetainedSizeInBytes(); + assertThat(retainedBefore).isGreaterThan(1_000_000); + + long id0; + long id1; + try (RowReferencePageManager.LoadCursor cursor = pageManager.add(page)) { + assertThat(cursor.advance()).isTrue(); + id0 = cursor.allocateRowId(); + assertThat(cursor.advance()).isTrue(); + id1 = cursor.allocateRowId(); + } + assertThat(pageManager.getPageBytes()).isGreaterThanOrEqualTo(retainedBefore); + + // Fully referenced page is still a compaction candidate because of the slack + assertThat(pageManager.getCompactionCandidateCount()).isEqualTo(1); + pageManager.compactIfNeeded(); + assertThat(pageManager.getCompactionCandidateCount()).isEqualTo(0); + assertThat(pageManager.getPageBytes()).isLessThan(retainedBefore / 100); + assertThat(extractString(pageManager, id0)).isEqualTo("a"); + assertThat(extractString(pageManager, id1)).isEqualTo("b"); + + pageManager.dereference(id0); + assertThat(pageManager.getCompactionCandidateCount()).isEqualTo(0); + assertThat(extractString(pageManager, id1)).isEqualTo("b"); + + pageManager.dereference(id1); + assertThat(pageManager.getPageBytes()).isEqualTo(0); + } + + @Test + public void testBuilderResetSkewCompaction() + { + RowReferencePageManager pageManager = new RowReferencePageManager(); + + // Builder sized to 1.25x the value, matching the block builder reset skew + int valueLength = 1_000_000; + BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 1, valueLength * 5 / 4); + VARCHAR.writeString(blockBuilder, "z".repeat(valueLength)); + Page page = new Page(blockBuilder.build()); + long retainedBefore = page.getRetainedSizeInBytes(); + assertThat(retainedBefore).isGreaterThan(valueLength * 5L / 4); + + long id; + try (RowReferencePageManager.LoadCursor cursor = pageManager.add(page)) { + assertThat(cursor.advance()).isTrue(); + id = cursor.allocateRowId(); + } + assertThat(pageManager.getPageBytes()).isGreaterThanOrEqualTo(retainedBefore); + + assertThat(pageManager.getCompactionCandidateCount()).isEqualTo(1); + pageManager.compactIfNeeded(); + assertThat(pageManager.getCompactionCandidateCount()).isEqualTo(0); + assertThat(pageManager.getPageBytes()).isLessThan(valueLength + 4096); + assertThat(extractString(pageManager, id)).hasSize(valueLength); + } + + @Test + public void testSharedDictionaryPages() + { + RowReferencePageManager pageManager = new RowReferencePageManager(); + + BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, 2); + VARCHAR.writeString(blockBuilder, "x".repeat(100_000)); + VARCHAR.writeString(blockBuilder, "y".repeat(100_000)); + Block dictionary = blockBuilder.build(); + Page page0 = new Page(DictionaryBlock.create(1, dictionary, new int[] {0})); + Page page1 = new Page(DictionaryBlock.create(1, dictionary, new int[] {1})); + + long id0; + long id1; + try (RowReferencePageManager.LoadCursor cursor = pageManager.add(page0)) { + assertThat(cursor.advance()).isTrue(); + id0 = cursor.allocateRowId(); + } + try (RowReferencePageManager.LoadCursor cursor = pageManager.add(page1)) { + assertThat(cursor.advance()).isTrue(); + id1 = cursor.allocateRowId(); + } + + // Each page is charged for the whole shared dictionary until it is copied + long pageBytesBeforeCompaction = pageManager.getPageBytes(); + assertThat(pageBytesBeforeCompaction).isGreaterThanOrEqualTo(2 * dictionary.getRetainedSizeInBytes()); + assertThat(pageManager.getCompactionCandidateCount()).isEqualTo(2); + pageManager.compactIfNeeded(); + assertThat(pageManager.getCompactionCandidateCount()).isEqualTo(0); + assertThat(pageManager.getPageBytes()).isLessThan(pageBytesBeforeCompaction / 2 + 1024); + assertThat(extractString(pageManager, id0)).isEqualTo("x".repeat(100_000)); + assertThat(extractString(pageManager, id1)).isEqualTo("y".repeat(100_000)); + } + + private static String extractString(RowReferencePageManager pageManager, long rowId) + { + Page page = pageManager.getPage(rowId); + int position = pageManager.getPosition(rowId); + return VARCHAR.getSlice(page.getBlock(0), position).toStringUtf8(); + } + private static long extractValue(RowReferencePageManager pageManager, long rowId) { Page page = pageManager.getPage(rowId); From 34754434bf26e9fd1b4c0ebd0ab0a45c949d2871 Mon Sep 17 00:00:00 2001 From: Dain Sundstrom Date: Sun, 12 Jul 2026 20:33:16 -0700 Subject: [PATCH 12/18] Offer filter selections to source pages Filters finish before projections are evaluated, but keeping their row selection entirely in PageProcessor prevents capable lazy sources from avoiding work for rejected projected values. Applying every selection is also unsafe: when a source must decode the full block, remapping the page can replace compact projection output with a slower indirect representation. Offer the selection as an atomic operation. Sources accept only when they can avoid loading or decoding data; rejection preserves the existing projection path. Materialize known filter inputs before entering generated code so reader implementations do not interfere with optimizing the filter loop. --- .../trino/operator/project/InputChannels.java | 5 + .../trino/operator/project/PageProcessor.java | 24 +++ .../gen/columnar/ColumnarFilterEvaluator.java | 5 +- .../io/trino/operator/TestingSourcePage.java | 3 +- .../operator/project/TestPageProcessor.java | 161 ++++++++++++++++++ .../columnar/TestColumnarFilterEvaluator.java | 89 ++++++++++ .../MemoryUsageReportingPageSource.java | 8 + .../io/trino/spi/connector/SourcePage.java | 13 ++ .../trino/plugin/base/MappedPageSource.java | 6 + .../hive/TransformConnectorPageSource.java | 22 ++- .../TestTransformConnectorPageSource.java | 157 +++++++++++++++++ .../iceberg/IcebergPageSourceProvider.java | 6 + 12 files changed, 495 insertions(+), 4 deletions(-) create mode 100644 core/trino-main/src/test/java/io/trino/sql/gen/columnar/TestColumnarFilterEvaluator.java create mode 100644 plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestTransformConnectorPageSource.java diff --git a/core/trino-main/src/main/java/io/trino/operator/project/InputChannels.java b/core/trino-main/src/main/java/io/trino/operator/project/InputChannels.java index 790c815938dd..f87e982b40f7 100644 --- a/core/trino-main/src/main/java/io/trino/operator/project/InputChannels.java +++ b/core/trino-main/src/main/java/io/trino/operator/project/InputChannels.java @@ -182,6 +182,11 @@ public Page getColumns(int[] channels) public void selectPositions(int[] positions, int offset, int size) { sourcePage.selectPositions(positions, offset, size); + selectLoadedBlocks(positions, offset, size); + } + + private void selectLoadedBlocks(int[] positions, int offset, int size) + { for (int i = 0; i < blocks.length; i++) { Block block = blocks[i]; if (block != null) { diff --git a/core/trino-main/src/main/java/io/trino/operator/project/PageProcessor.java b/core/trino-main/src/main/java/io/trino/operator/project/PageProcessor.java index 8c27256d180b..e8eef3376d4b 100644 --- a/core/trino-main/src/main/java/io/trino/operator/project/PageProcessor.java +++ b/core/trino-main/src/main/java/io/trino/operator/project/PageProcessor.java @@ -135,9 +135,33 @@ public WorkProcessor createWorkProcessor( return WorkProcessor.of(new Page(selectedPositions.size())); } + if (!isAllPositions(selectedPositions, page.getPositionCount())) { + int[] positions; + int positionsOffset; + if (selectedPositions.isList()) { + positions = selectedPositions.getPositions(); + positionsOffset = selectedPositions.getOffset(); + } + else { + positions = new int[selectedPositions.size()]; + positionsOffset = 0; + for (int index = 0; index < positions.length; index++) { + positions[index] = selectedPositions.getOffset() + index; + } + } + if (page.trySelectPositions(positions, positionsOffset, selectedPositions.size())) { + selectedPositions = positionsRange(0, selectedPositions.size()); + } + } + return WorkProcessor.create(new ProjectSelectedPositions(session, memoryContext, metrics, page, selectedPositions)); } + private static boolean isAllPositions(SelectedPositions selectedPositions, int positionCount) + { + return !selectedPositions.isList() && selectedPositions.getOffset() == 0 && selectedPositions.size() == positionCount; + } + private class ProjectSelectedPositions implements WorkProcessor.Process { diff --git a/core/trino-main/src/main/java/io/trino/sql/gen/columnar/ColumnarFilterEvaluator.java b/core/trino-main/src/main/java/io/trino/sql/gen/columnar/ColumnarFilterEvaluator.java index f2444c69925d..b42964b4c3a4 100644 --- a/core/trino-main/src/main/java/io/trino/sql/gen/columnar/ColumnarFilterEvaluator.java +++ b/core/trino-main/src/main/java/io/trino/sql/gen/columnar/ColumnarFilterEvaluator.java @@ -38,8 +38,9 @@ public SelectionResult evaluate(ConnectorSession session, SelectedPositions acti if (activePositions.isEmpty()) { return new SelectionResult(activePositions, 0); } - // Should load only the blocks necessary for evaluating the kernel and unwrap lazy blocks - SourcePage loadedPage = filter.getInputChannels().getInputChannels(page); + // Materialize only the filter inputs here so reader code does not consume + // the generated filter's JIT inlining budget. + SourcePage loadedPage = SourcePage.create(filter.getInputChannels().getInputChannels(page).getPage()); if (outputPositions.length < activePositions.size()) { outputPositions = new int[activePositions.size()]; } diff --git a/core/trino-main/src/test/java/io/trino/operator/TestingSourcePage.java b/core/trino-main/src/test/java/io/trino/operator/TestingSourcePage.java index c873b90d54b9..e0a139ea6c7d 100644 --- a/core/trino-main/src/test/java/io/trino/operator/TestingSourcePage.java +++ b/core/trino-main/src/test/java/io/trino/operator/TestingSourcePage.java @@ -30,7 +30,7 @@ public class TestingSourcePage { private static final long INSTANCE_SIZE = instanceSize(TestingSourcePage.class); - private final int positionCount; + private int positionCount; private final Block[] blocks; private final boolean[] loaded; @@ -127,5 +127,6 @@ public void selectPositions(int[] positions, int offset, int size) blocks[i] = block.getPositions(positions, offset, size); } } + positionCount = size; } } diff --git a/core/trino-main/src/test/java/io/trino/operator/project/TestPageProcessor.java b/core/trino-main/src/test/java/io/trino/operator/project/TestPageProcessor.java index 843ae84aca9e..2fe3f4e0c45f 100644 --- a/core/trino-main/src/test/java/io/trino/operator/project/TestPageProcessor.java +++ b/core/trino-main/src/test/java/io/trino/operator/project/TestPageProcessor.java @@ -22,6 +22,7 @@ import io.trino.spi.Page; import io.trino.spi.block.Block; import io.trino.spi.block.DictionaryBlock; +import io.trino.spi.block.LongArrayBlock; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.SourcePage; import io.trino.sql.gen.columnar.PageFilterEvaluator; @@ -125,6 +126,166 @@ public void testPartialFilter() assertPageEquals(ImmutableList.of(BIGINT), outputPage, new Page(createLongSequenceBlock(25, 75))); } + @Test + public void testFilterPushesSelectionIntoLazySourcePage() + { + int[] selectedPositions = {1, 3, 5}; + PageFilter filter = new PageFilter() + { + @Override + public boolean isDeterministic() + { + return true; + } + + @Override + public InputChannels getInputChannels() + { + return new InputChannels(0); + } + + @Override + public SelectedPositions filter(ConnectorSession session, SourcePage page) + { + page.getBlock(0); + return positionsList(selectedPositions, 0, selectedPositions.length); + } + }; + PageProcessor pageProcessor = new PageProcessor( + Optional.of(new PageFilterEvaluator(filter)), + Optional.empty(), + ImmutableList.of(new InputPageProjection(1)), + OptionalInt.of(MAX_BATCH_SIZE)); + + TestingSourcePage inputPage = new TestingSourcePage(10, createLongSequenceBlock(0, 10), createLongSequenceBlock(100, 110)) + { + @Override + public boolean trySelectPositions(int[] positions, int offset, int size) + { + int[] retainedPositions = Arrays.copyOfRange(positions, offset, offset + size); + selectPositions(retainedPositions, 0, size); + return true; + } + + @Override + public void selectPositions(int[] positions, int offset, int size) + { + assertThat(wasLoaded(0)).isTrue(); + assertThat(wasLoaded(1)).isFalse(); + super.selectPositions(positions, offset, size); + } + }; + + Iterator> output = processAndAssertRetainedPageSize(pageProcessor, inputPage); + Page outputPage = getOnlyElement(ImmutableList.copyOf(output)).orElseThrow(); + Arrays.fill(selectedPositions, 0); + assertPageEquals(ImmutableList.of(BIGINT), outputPage, new Page(createLongSequenceBlock(101, 106).copyPositions(new int[] {0, 2, 4}, 0, 3))); + assertThat(inputPage.getPositionCount()).isEqualTo(3); + assertThat(inputPage.wasLoaded(1)).isTrue(); + } + + @Test + public void testInputChannelsSelectsLoadedBlocks() + { + SourcePage page = new InputChannels(1).getInputChannels( + new TestingSourcePage(10, createLongSequenceBlock(0, 10), createLongSequenceBlock(100, 110))); + page.getBlock(0); + + int[] positions = {1, 3, 5}; + page.selectPositions(positions, 0, positions.length); + + assertPageEquals( + ImmutableList.of(BIGINT), + page.getPage(), + new Page(createLongSequenceBlock(100, 110).copyPositions(positions, 0, positions.length))); + } + + @Test + public void testFilterPushesRangeSelectionIntoLazySourcePage() + { + PageFilter filter = new PageFilter() + { + @Override + public boolean isDeterministic() + { + return true; + } + + @Override + public InputChannels getInputChannels() + { + return new InputChannels(0); + } + + @Override + public SelectedPositions filter(ConnectorSession session, SourcePage page) + { + page.getBlock(0); + return positionsRange(2, 3); + } + }; + PageProcessor pageProcessor = new PageProcessor( + Optional.of(new PageFilterEvaluator(filter)), + Optional.empty(), + ImmutableList.of(new InputPageProjection(1)), + OptionalInt.of(MAX_BATCH_SIZE)); + + TestingSourcePage inputPage = new TestingSourcePage(10, createLongSequenceBlock(0, 10), createLongSequenceBlock(100, 110)) + { + @Override + public boolean trySelectPositions(int[] positions, int offset, int size) + { + int[] retainedPositions = Arrays.copyOfRange(positions, offset, offset + size); + selectPositions(retainedPositions, 0, size); + return true; + } + }; + + Iterator> output = processAndAssertRetainedPageSize(pageProcessor, inputPage); + Page outputPage = getOnlyElement(ImmutableList.copyOf(output)).orElseThrow(); + assertPageEquals(ImmutableList.of(BIGINT), outputPage, new Page(createLongSequenceBlock(102, 105))); + assertThat(inputPage.getPositionCount()).isEqualTo(3); + } + + @Test + public void testFilterUsesEngineSelectionWhenSourceRejectsPushdown() + { + PageFilter filter = new PageFilter() + { + @Override + public boolean isDeterministic() + { + return true; + } + + @Override + public InputChannels getInputChannels() + { + return new InputChannels(0); + } + + @Override + public SelectedPositions filter(ConnectorSession session, SourcePage page) + { + page.getBlock(0); + return positionsList(new int[] {1, 3, 5}, 0, 3); + } + }; + PageProcessor pageProcessor = new PageProcessor( + Optional.of(new PageFilterEvaluator(filter)), + Optional.empty(), + ImmutableList.of(new InputPageProjection(1)), + OptionalInt.of(MAX_BATCH_SIZE)); + + TestingSourcePage inputPage = new TestingSourcePage(10, createLongSequenceBlock(0, 10), createLongSequenceBlock(100, 110)); + + Iterator> output = processAndAssertRetainedPageSize(pageProcessor, inputPage); + Page outputPage = getOnlyElement(ImmutableList.copyOf(output)).orElseThrow(); + assertPageEquals(ImmutableList.of(BIGINT), outputPage, new Page(createLongSequenceBlock(101, 106).copyPositions(new int[] {0, 2, 4}, 0, 3))); + assertThat(outputPage.getBlock(0)).isInstanceOf(LongArrayBlock.class); + assertThat(inputPage.getPositionCount()).isEqualTo(10); + } + @Test public void testSelectAllFilter() { diff --git a/core/trino-main/src/test/java/io/trino/sql/gen/columnar/TestColumnarFilterEvaluator.java b/core/trino-main/src/test/java/io/trino/sql/gen/columnar/TestColumnarFilterEvaluator.java new file mode 100644 index 000000000000..ad7c009e40b6 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/sql/gen/columnar/TestColumnarFilterEvaluator.java @@ -0,0 +1,89 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.gen.columnar; + +import io.trino.operator.TestingSourcePage; +import io.trino.operator.project.InputChannels; +import io.trino.operator.project.SelectedPositions; +import io.trino.spi.block.Block; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.SourcePage; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static io.trino.block.BlockAssertions.createLongSequenceBlock; +import static io.trino.operator.project.SelectedPositions.positionsList; +import static io.trino.operator.project.SelectedPositions.positionsRange; +import static io.trino.testing.TestingConnectorSession.SESSION; +import static java.lang.System.arraycopy; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestColumnarFilterEvaluator +{ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testMaterializesOnlyFilterInputs(boolean list) + { + Block first = createLongSequenceBlock(0, 4); + Block last = createLongSequenceBlock(100, 104); + TestingSourcePage source = new TestingSourcePage(4, first, null, last); + ColumnarFilter filter = new ColumnarFilter() + { + @Override + public InputChannels getInputChannels() + { + return new InputChannels(2, 0); + } + + @Override + public int filterPositionsRange(ConnectorSession session, int[] outputPositions, int offset, int size, SourcePage loadedPage) + { + assertInputsLoaded(loadedPage); + for (int index = 0; index < size; index++) { + outputPositions[index] = offset + index; + } + return size; + } + + @Override + public int filterPositionsList(ConnectorSession session, int[] outputPositions, int[] activePositions, int offset, int size, SourcePage loadedPage) + { + assertInputsLoaded(loadedPage); + arraycopy(activePositions, offset, outputPositions, 0, size); + return size; + } + + private void assertInputsLoaded(SourcePage loadedPage) + { + assertThat(source.wasLoaded(0)).isTrue(); + assertThat(source.wasLoaded(1)).isFalse(); + assertThat(source.wasLoaded(2)).isTrue(); + assertThat(loadedPage.getPositionCount()).isEqualTo(4); + assertThat(loadedPage.getChannelCount()).isEqualTo(2); + assertThat(loadedPage.getBlock(0)).isSameAs(last); + assertThat(loadedPage.getBlock(1)).isSameAs(first); + } + }; + SelectedPositions activePositions = list ? positionsList(new int[] {0, 1, 3}, 1, 2) : positionsRange(1, 2); + SelectedPositions result = new ColumnarFilterEvaluator(filter).evaluate(SESSION, activePositions, source).selectedPositions(); + assertThat(result.size()).isEqualTo(2); + assertThat(result.isList()).isEqualTo(list); + if (list) { + assertThat(result.getPositions()).containsExactly(1, 3); + } + else { + assertThat(result.getOffset()).isEqualTo(1); + } + } +} diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/MemoryUsageReportingPageSource.java b/core/trino-spi/src/main/java/io/trino/spi/connector/MemoryUsageReportingPageSource.java index 79793b15d04b..f291d807521a 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/MemoryUsageReportingPageSource.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/MemoryUsageReportingPageSource.java @@ -155,6 +155,14 @@ public Page getPage() return page; } + @Override + public boolean trySelectPositions(int[] positions, int offset, int size) + { + boolean selected = sourcePage.trySelectPositions(positions, offset, size); + memoryContext.setBytes(pageSource.getMemoryUsage()); + return selected; + } + @Override public Page getColumns(int[] channels) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/SourcePage.java b/core/trino-spi/src/main/java/io/trino/spi/connector/SourcePage.java index 37316e2331d5..2352317c00b4 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/SourcePage.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/SourcePage.java @@ -86,6 +86,19 @@ static SourcePage create(Page page) */ Page getPage(); + /** + * Attempts to apply the specified position selection when doing so can avoid loading or + * decoding data that has not been accessed yet. Returns whether the selection was applied. + * If this method returns false, the logical contents and position count of the page must remain unchanged. + *

    + * The positions array is owned by the caller and may only be accessed for the duration of + * this call. An implementation must copy any positions it retains. + */ + default boolean trySelectPositions(int[] positions, int offset, int size) + { + return false; + } + /** * Gets a projection of the page containing only the specified channels. */ diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/MappedPageSource.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/MappedPageSource.java index 388cedf0f081..b61d4df676e9 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/MappedPageSource.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/MappedPageSource.java @@ -137,6 +137,12 @@ public Page getPage() return sourcePage.getColumns(channels); } + @Override + public boolean trySelectPositions(int[] positions, int offset, int size) + { + return sourcePage.trySelectPositions(positions, offset, size); + } + @Override public Page getColumns(int[] channels) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TransformConnectorPageSource.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TransformConnectorPageSource.java index 771bbf927bee..f6e119f36c06 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TransformConnectorPageSource.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/TransformConnectorPageSource.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.OptionalLong; @@ -368,14 +369,33 @@ public Page getPage() return new Page(getPositionCount(), blocks); } + @Override + public boolean trySelectPositions(int[] positions, int offset, int size) + { + if (!sourcePage.trySelectPositions(positions, offset, size)) { + return false; + } + selectLoadedBlocks(positions, offset, size); + return true; + } + @Override public void selectPositions(int[] positions, int offset, int size) { sourcePage.selectPositions(positions, offset, size); + selectLoadedBlocks(positions, offset, size); + } + + private void selectLoadedBlocks(int[] positions, int offset, int size) + { + int[] retainedPositions = null; for (int i = 0; i < blocks.length; i++) { Block block = blocks[i]; if (block != null) { - blocks[i] = block.getPositions(positions, offset, size); + if (retainedPositions == null) { + retainedPositions = Arrays.copyOfRange(positions, offset, offset + size); + } + blocks[i] = block.getPositions(retainedPositions, 0, size); } } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestTransformConnectorPageSource.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestTransformConnectorPageSource.java new file mode 100644 index 000000000000..98d2fb09d9b6 --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestTransformConnectorPageSource.java @@ -0,0 +1,157 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive; + +import io.trino.spi.Page; +import io.trino.spi.block.Block; +import io.trino.spi.block.LongArrayBlock; +import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.connector.SourcePage; +import org.junit.jupiter.api.Test; + +import java.util.Optional; +import java.util.function.ObjLongConsumer; + +import static io.trino.spi.type.BigintType.BIGINT; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestTransformConnectorPageSource +{ + @Test + public void testSelectedPositionsRemapLoadedBlockWithoutRetainingCallerArray() + { + SourcePage inputPage = new SelectingSourcePage(new LongArrayBlock(5, Optional.empty(), new long[] {10, 11, 12, 13, 14})); + ConnectorPageSource pageSource = TransformConnectorPageSource.builder() + .transform(0, block -> block) + .build(new SinglePageSource(inputPage)); + + SourcePage page = pageSource.getNextSourcePage(); + assertThat(page.getBlock(0).getPositionCount()).isEqualTo(5); + + int[] positions = {99, 3, 1, 99}; + assertThat(page.trySelectPositions(positions, 1, 2)).isTrue(); + positions[1] = 0; + positions[2] = 0; + + assertThat(page.getPositionCount()).isEqualTo(2); + assertThat(BIGINT.getLong(page.getBlock(0), 0)).isEqualTo(13); + assertThat(BIGINT.getLong(page.getBlock(0), 1)).isEqualTo(11); + } + + private static final class SelectingSourcePage + implements SourcePage + { + private Block block; + + private SelectingSourcePage(Block block) + { + this.block = block; + } + + @Override + public int getPositionCount() + { + return block.getPositionCount(); + } + + @Override + public long getSizeInBytes() + { + return block.getSizeInBytes(); + } + + @Override + public long getRetainedSizeInBytes() + { + return block.getRetainedSizeInBytes(); + } + + @Override + public void retainedBytesForEachPart(ObjLongConsumer consumer) + { + block.retainedBytesForEachPart(consumer); + } + + @Override + public int getChannelCount() + { + return 1; + } + + @Override + public Block getBlock(int channel) + { + return block; + } + + @Override + public Page getPage() + { + return new Page(block); + } + + @Override + public boolean trySelectPositions(int[] positions, int offset, int size) + { + block = block.copyPositions(positions, offset, size); + return true; + } + + @Override + public void selectPositions(int[] positions, int offset, int size) + { + block = block.copyPositions(positions, offset, size); + } + } + + private static final class SinglePageSource + implements ConnectorPageSource + { + private SourcePage page; + + private SinglePageSource(SourcePage page) + { + this.page = page; + } + + @Override + public long getCompletedBytes() + { + return 0; + } + + @Override + public long getReadTimeNanos() + { + return 0; + } + + @Override + public boolean isFinished() + { + return page == null; + } + + @Override + public SourcePage getNextSourcePage() + { + SourcePage result = page; + page = null; + return result; + } + + @Override + public void close() {} + } +} diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java index f00cde21ca35..a8a287ad3dc2 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java @@ -2210,6 +2210,12 @@ public Page getPage() return sourcePage.getColumns(channels); } + @Override + public boolean trySelectPositions(int[] positions, int offset, int size) + { + return sourcePage.trySelectPositions(positions, offset, size); + } + @Override public Page getColumns(int[] channels) { From 1be51b1e350590195947d2142ed1a1919c3abb98 Mon Sep 17 00:00:00 2001 From: Dain Sundstrom Date: Sun, 12 Jul 2026 20:36:52 -0700 Subject: [PATCH 13/18] Read selected Parquet values Masking a fully decoded block preserves semantics, but it does not provide late materialization because rejected values still incur decode and allocation costs. Allow flat primitive readers to consume ascending selected positions directly. Unsupported readers and selection shapes retain the existing full-read path. --- .../io/trino/parquet/reader/ColumnReader.java | 10 + .../parquet/reader/flat/FlatColumnReader.java | 124 ++++++++- .../reader/flat/TestFlatColumnReader.java | 245 ++++++++++++++++++ 3 files changed, 372 insertions(+), 7 deletions(-) diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ColumnReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ColumnReader.java index e06686ba7ec6..e6a42de857a5 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ColumnReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ColumnReader.java @@ -24,4 +24,14 @@ public interface ColumnReader void prepareNextRead(int batchSize); ColumnChunk readPrimitive(); + + default boolean supportsSelectedPositions() + { + return false; + } + + default ColumnChunk readPrimitive(int[] positions, int offset, int positionCount) + { + throw new UnsupportedOperationException("Selected positions are not supported"); + } } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/flat/FlatColumnReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/flat/FlatColumnReader.java index 36d71b7a96b4..81bbe319472b 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/flat/FlatColumnReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/flat/FlatColumnReader.java @@ -38,7 +38,9 @@ import static io.trino.parquet.ParquetEncoding.RLE; import static io.trino.spi.block.Bitmap.setBits; import static io.trino.spi.block.Bitmap.wordsForBits; +import static java.lang.Math.min; import static java.lang.Math.toIntExact; +import static java.util.Objects.checkFromIndexSize; import static java.util.Objects.requireNonNull; public class FlatColumnReader @@ -100,6 +102,31 @@ public ColumnChunk readPrimitive() return columnChunk; } + @Override + public boolean supportsSelectedPositions() + { + return true; + } + + @Override + public ColumnChunk readPrimitive(int[] positions, int offset, int positionCount) + { + checkFromIndexSize(offset, positionCount, positions.length); + + seek(); + ColumnChunk columnChunk; + if (isNonNull()) { + columnChunk = readNonNull(positions, offset, positionCount); + } + else { + columnChunk = readNullable(positions, offset, positionCount); + } + + readOffset = 0; + nextBatchSize = 0; + return columnChunk; + } + @Override public void prepareNextRead(int batchSize) { @@ -112,7 +139,12 @@ private void seek() if (readOffset > 0) { log.debug("seek field %s, readOffset %d, remainingPageValueCount %d", field, readOffset, remainingPageValueCount); } - int remainingInBatch = readOffset; + skipRows(readOffset); + } + + private void skipRows(int rowCount) + { + int remainingInBatch = rowCount; while (remainingInBatch > 0) { if (remainingPageValueCount == 0) { remainingInBatch = seekToNextPage(remainingInBatch); @@ -176,13 +208,36 @@ else if (nonNullCount < chunkSize && !validityMaterialized) { return valuesBuffer.createNullableBlock(valueIsValid, field.getType()); } + private ColumnChunk readNullable(int[] positions, int offset, int positionCount) + { + log.debug("readNullable selected field %s, nextBatchSize %d, positionCount %d, remainingPageValueCount %d", field, nextBatchSize, positionCount, remainingPageValueCount); + NullableValuesBuffer valuesBuffer = createNullableValuesBuffer(positionCount); + long[] valueIsValid = new long[wordsForBits(positionCount)]; + readSelectedPositions(positions, offset, positionCount, (outputOffset, runLength) -> readNullableRows(valuesBuffer, valueIsValid, outputOffset, runLength)); + return valuesBuffer.createNullableBlock(valueIsValid, field.getType()); + } + @VisibleForTesting ColumnChunk readNonNull() { log.debug("readNonNull field %s, nextBatchSize %d, remainingPageValueCount %d", field, nextBatchSize, remainingPageValueCount); NonNullValuesBuffer valuesBuffer = createNonNullValuesBuffer(nextBatchSize); - int remainingInBatch = nextBatchSize; - int offset = 0; + readNonNullRows(valuesBuffer, 0, nextBatchSize); + return valuesBuffer.createNonNullBlock(field.getType()); + } + + private ColumnChunk readNonNull(int[] positions, int offset, int positionCount) + { + log.debug("readNonNull selected field %s, nextBatchSize %d, positionCount %d, remainingPageValueCount %d", field, nextBatchSize, positionCount, remainingPageValueCount); + NonNullValuesBuffer valuesBuffer = createNonNullValuesBuffer(positionCount); + readSelectedPositions(positions, offset, positionCount, (outputOffset, runLength) -> readNonNullRows(valuesBuffer, outputOffset, runLength)); + return valuesBuffer.createNonNullBlock(field.getType()); + } + + private void readNonNullRows(NonNullValuesBuffer valuesBuffer, int offset, int rowCount) + { + int remainingInBatch = rowCount; + int outputOffset = offset; while (remainingInBatch > 0) { if (remainingPageValueCount == 0) { if (!readNextPage()) { @@ -193,14 +248,64 @@ ColumnChunk readNonNull() if (skipToRowRangesStart()) { continue; } - int chunkSize = rowRanges.advanceRange(Math.min(remainingPageValueCount, remainingInBatch)); + int chunkSize = rowRanges.advanceRange(min(remainingPageValueCount, remainingInBatch)); - valuesBuffer.readNonNullValues(valueDecoder, offset, chunkSize); - offset += chunkSize; + valuesBuffer.readNonNullValues(valueDecoder, outputOffset, chunkSize); + outputOffset += chunkSize; remainingInBatch -= chunkSize; remainingPageValueCount -= chunkSize; } - return valuesBuffer.createNonNullBlock(field.getType()); + } + + private void readNullableRows(NullableValuesBuffer valuesBuffer, long[] valueIsValid, int offset, int rowCount) + { + int remainingInBatch = rowCount; + int outputOffset = offset; + while (remainingInBatch > 0) { + if (remainingPageValueCount == 0) { + if (!readNextPage()) { + throwEndOfBatchException(remainingInBatch); + } + } + + if (skipToRowRangesStart()) { + continue; + } + int chunkSize = rowRanges.advanceRange(min(remainingPageValueCount, remainingInBatch)); + int nonNullCount = definitionLevelDecoder.readNext(valueIsValid, outputOffset, chunkSize); + if (nonNullCount == chunkSize) { + setBits(valueIsValid, 0, outputOffset, chunkSize); + } + + valuesBuffer.readNullableValues(valueDecoder, valueIsValid, outputOffset, nonNullCount, chunkSize); + + outputOffset += chunkSize; + remainingInBatch -= chunkSize; + remainingPageValueCount -= chunkSize; + } + } + + private void readSelectedPositions(int[] positions, int offset, int positionCount, SelectedPositionsReader selectedPositionsReader) + { + int batchPosition = 0; + int outputOffset = 0; + int endOffset = offset + positionCount; + for (int positionOffset = offset; positionOffset < endOffset; positionOffset++) { + int position = positions[positionOffset]; + checkArgument(position >= batchPosition, "positions must be strictly ascending"); + checkArgument(position < nextBatchSize, "position %s is outside of batch size %s", position, nextBatchSize); + skipRows(position - batchPosition); + + int runLength = 1; + while (positionOffset + runLength < endOffset && positions[positionOffset + runLength] == position + runLength) { + runLength++; + } + selectedPositionsReader.read(outputOffset, runLength); + positionOffset += runLength - 1; + batchPosition = position + runLength; + outputOffset += runLength; + } + skipRows(nextBatchSize - batchPosition); } /** @@ -285,6 +390,11 @@ else if (page instanceof DataPageV2 dataPageV2) { return page; } + private interface SelectedPositionsReader + { + void read(int outputOffset, int runLength); + } + private void readFlatPageV1(DataPageV1 page) { Slice buffer = page.getSlice(); diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/flat/TestFlatColumnReader.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/flat/TestFlatColumnReader.java index f83469bc2cd6..d248a31c393a 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/flat/TestFlatColumnReader.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/flat/TestFlatColumnReader.java @@ -17,6 +17,7 @@ import io.airlift.slice.Slices; import io.trino.parquet.DataPage; import io.trino.parquet.DataPageV1; +import io.trino.parquet.DictionaryPage; import io.trino.parquet.ParquetDataSourceId; import io.trino.parquet.ParquetEncoding; import io.trino.parquet.ParquetReaderOptions; @@ -25,28 +26,40 @@ import io.trino.parquet.reader.ColumnReader; import io.trino.parquet.reader.ColumnReaderFactory; import io.trino.parquet.reader.PageReader; +import io.trino.parquet.reader.TestingColumnReader.ColumnReaderFormat; +import io.trino.parquet.reader.TestingColumnReader.DataPageVersion; import io.trino.spi.block.Block; import org.apache.parquet.bytes.HeapByteBufferAllocator; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.values.ValuesWriter; +import org.apache.parquet.column.values.dictionary.DictionaryValuesWriter; import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridValuesWriter; import org.apache.parquet.schema.PrimitiveType; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; +import java.util.Random; import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static io.trino.parquet.ParquetEncoding.BIT_PACKED; import static io.trino.parquet.ParquetEncoding.PLAIN; import static io.trino.parquet.ParquetEncoding.RLE; +import static io.trino.parquet.ParquetEncoding.RLE_DICTIONARY; +import static io.trino.parquet.reader.TestingColumnReader.DataPageVersion.V1; +import static io.trino.parquet.reader.TestingColumnReader.getDictionaryPage; import static io.trino.parquet.reader.TestingValuesWriters.getValuesWriter; import static io.trino.spi.type.IntegerType.INTEGER; import static org.apache.parquet.format.CompressionCodec.UNCOMPRESSED; import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; +import static org.apache.parquet.schema.Type.Repetition.OPTIONAL; import static org.apache.parquet.schema.Type.Repetition.REQUIRED; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -59,7 +72,9 @@ public class TestFlatColumnReader extends AbstractColumnReaderTest { private static final PrimitiveType TYPE = new PrimitiveType(REQUIRED, INT32, ""); + private static final PrimitiveType OPTIONAL_TYPE = new PrimitiveType(OPTIONAL, INT32, ""); private static final PrimitiveField NULLABLE_FIELD = new PrimitiveField(INTEGER, false, new ColumnDescriptor(new String[] {"test"}, TYPE, 0, 0), 0); + private static final PrimitiveField OPTIONAL_FIELD = new PrimitiveField(INTEGER, false, new ColumnDescriptor(new String[] {"test"}, OPTIONAL_TYPE, 0, 1), 0); private static final PrimitiveField FIELD = new PrimitiveField(INTEGER, true, new ColumnDescriptor(new String[] {"test"}, TYPE, 0, 0), 0); @Override @@ -125,6 +140,178 @@ public void testReadPageV1RleOnlyNulls() assertThat(block.isNull(0)).isTrue(); } + @Test + public void testReadSelectedPositions() + throws IOException + { + FlatColumnReader reader = (FlatColumnReader) createColumnReader(FIELD); + reader.setPageReader(getPlainPageReaderMock( + createPlainDataPage(0, 1, 2, 3, 4), + createPlainDataPage(5, 6, 7, 8, 9), + createPlainDataPage(10, 11, 12, 13, 14), + createPlainDataPage(15, 16)), + Optional.empty()); + + reader.prepareNextRead(15); + Block selected = reader.readPrimitive(new int[] {99, 1, 2, 11, 14, 99}, 1, 4).getBlock(); + assertThat(intValues(selected)).containsExactly(1, 2, 11, 14); + + reader.prepareNextRead(2); + assertThat(intValues(reader.readPrimitive().getBlock())).containsExactly(15, 16); + } + + @ParameterizedTest + @MethodSource("io.trino.parquet.reader.TestingColumnReader#readersWithPageVersions") + public void testReadSelectedPositionsAllTypes(DataPageVersion version, ColumnReaderFormat format) + throws IOException + { + PrimitiveField field = createField(format, true); + ColumnReader reader = createColumnReader(field); + ValuesWriter writer = format.getValuesWriter(version); + + T[] firstPageValues = format.write(writer, new Integer[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + DataPage firstPage = createDataPage(version, writer, field, new int[0], new int[10]); + T[] secondPageValues = format.resetAndWrite(writer, new Integer[] {10, 11, 12, 13, 14, 15, 16, 17, 18, 19}); + DataPage secondPage = createDataPage(version, writer, field, new int[0], new int[10]); + + reader.setPageReader(getPageReaderMock(List.of(firstPage, secondPage), null), Optional.empty()); + reader.prepareNextRead(20); + Block selected = reader.readPrimitive(new int[] {1, 2, 12, 19}, 0, 4).getBlock(); + + format.assertBlock(firstPageValues, selected, 1, 0, 2); + format.assertBlock(secondPageValues, selected, 2, 2, 1); + format.assertBlock(secondPageValues, selected, 9, 3, 1); + } + + @ParameterizedTest + @MethodSource("io.trino.parquet.reader.TestingColumnReader#readersWithPageVersions") + public void testReadSelectedNullablePositionsAllTypes(DataPageVersion version, ColumnReaderFormat format) + throws IOException + { + PrimitiveField field = createField(format, false); + ColumnReader reader = createColumnReader(field); + ValuesWriter writer = format.getValuesWriter(version); + T[] values = format.write(writer, new Integer[] {0, null, 2, 3, null, 5, 6, null, 8, 9}); + DataPage page = createNullableDataPage(version, writer, field, false, true, false, false, true, false, false, true, false, false); + + reader.setPageReader(getPageReaderMock(List.of(page), null), Optional.empty()); + reader.prepareNextRead(10); + Block selected = reader.readPrimitive(new int[] {1, 2, 4, 7, 9}, 0, 5).getBlock(); + + format.assertBlock(values, selected, 1, 0, 2); + format.assertBlock(values, selected, 4, 2, 1); + format.assertBlock(values, selected, 7, 3, 1); + format.assertBlock(values, selected, 9, 4, 1); + } + + @ParameterizedTest + @MethodSource("io.trino.parquet.reader.TestingColumnReader#dictionaryReadersWithPageVersions") + public void testReadSelectedDictionaryPositionsAllTypes(DataPageVersion version, ColumnReaderFormat format) + throws IOException + { + PrimitiveField field = createField(format, true); + ColumnReader reader = createColumnReader(field); + DictionaryValuesWriter writer = format.getDictionaryWriter(); + T[] values = format.write(writer, new Integer[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + DataPage page = createDataPage(version, RLE_DICTIONARY, writer, field, new int[0], new int[10]); + DictionaryPage dictionaryPage = getDictionaryPage(writer); + + reader.setPageReader(getPageReaderMock(List.of(page), dictionaryPage), Optional.empty()); + reader.prepareNextRead(10); + Block selected = reader.readPrimitive(new int[] {1, 2, 7, 9}, 0, 4).getBlock(); + + format.assertBlock(values, selected, 1, 0, 2); + format.assertBlock(values, selected, 7, 2, 1); + format.assertBlock(values, selected, 9, 3, 1); + } + + @Test + public void testReadEmptySelection() + throws IOException + { + FlatColumnReader reader = (FlatColumnReader) createColumnReader(FIELD); + reader.setPageReader(getPlainPageReaderMock( + createPlainDataPage(0, 1, 2, 3, 4), + createPlainDataPage(5, 6, 7, 8, 9), + createPlainDataPage(10, 11)), + Optional.empty()); + + reader.prepareNextRead(10); + assertThat(reader.readPrimitive(new int[0], 0, 0).getBlock().getPositionCount()).isZero(); + + reader.prepareNextRead(2); + assertThat(intValues(reader.readPrimitive().getBlock())).containsExactly(10, 11); + } + + @Test + public void testReadSelectedNullablePositions() + throws IOException + { + FlatColumnReader reader = (FlatColumnReader) createColumnReader(OPTIONAL_FIELD); + reader.setPageReader(getPlainPageReaderMock( + createNullablePage(0, null, 2, 3, null), + createNullablePage(5, null, 7, null, 9)), + Optional.empty()); + + reader.prepareNextRead(10); + Block selected = reader.readPrimitive(new int[] {1, 2, 3, 6, 8, 9}, 0, 6).getBlock(); + assertThat(nullableIntValues(selected)).containsExactly(null, 2, 3, null, null, 9); + } + + @Test + public void testReadSelectedDictionaryPositions() + throws IOException + { + DictionaryValuesWriter writer = (DictionaryValuesWriter) getValuesWriter(RLE_DICTIONARY, INT32, OptionalInt.empty()); + for (int value : new int[] {10, 11, 10, 12, 11, 13, 10, 13}) { + writer.writeInteger(value); + } + DataPage page = createDataPage(V1, RLE_DICTIONARY, writer, FIELD, new int[0], new int[8]); + DictionaryPage dictionaryPage = getDictionaryPage(writer); + + FlatColumnReader reader = (FlatColumnReader) createColumnReader(FIELD); + reader.setPageReader(getPageReaderMock(List.of(page), dictionaryPage), Optional.empty()); + reader.prepareNextRead(8); + + Block selected = reader.readPrimitive(new int[] {0, 2, 3, 6, 7}, 0, 5).getBlock(); + assertThat(intValues(selected)).containsExactly(10, 10, 12, 10, 13); + } + + @Test + public void testReadSelectedPositionsRandomized() + throws IOException + { + int valueCount = 257; + int[] values = new int[valueCount]; + for (int index = 0; index < valueCount; index++) { + values[index] = index * 17; + } + + Random random = new Random(918273645L); + for (int iteration = 0; iteration < 50; iteration++) { + int[] positions = random.ints(valueCount, 0, valueCount) + .distinct() + .sorted() + .limit(random.nextInt(valueCount + 1)) + .toArray(); + + FlatColumnReader reader = (FlatColumnReader) createColumnReader(FIELD); + reader.setPageReader(getPlainPageReaderMock( + createPlainDataPage(Arrays.copyOfRange(values, 0, 64)), + createPlainDataPage(Arrays.copyOfRange(values, 64, 129)), + createPlainDataPage(Arrays.copyOfRange(values, 129, 200)), + createPlainDataPage(Arrays.copyOfRange(values, 200, valueCount))), + Optional.empty()); + reader.prepareNextRead(valueCount); + + Block selected = reader.readPrimitive(positions, 0, positions.length).getBlock(); + assertThat(intValues(selected)).containsExactly(Arrays.stream(positions) + .map(position -> values[position]) + .boxed() + .toArray(Integer[]::new)); + } + } + private static PageReader getSimplePageReaderMock(ParquetEncoding encoding) throws IOException { @@ -143,6 +330,64 @@ private static PageReader getSimplePageReaderMock(ParquetEncoding encoding) return new PageReader(new ParquetDataSourceId("test"), UNCOMPRESSED, pages.iterator(), false, false, Optional.empty(), -1, -1); } + private static DataPage createPlainDataPage(int... values) + throws IOException + { + ValuesWriter writer = getValuesWriter(PLAIN, INT32, OptionalInt.empty()); + for (int value : values) { + writer.writeInteger(value); + } + byte[] valueBytes = writer.getBytes().toByteArray(); + return new DataPageV1( + Slices.wrappedBuffer(valueBytes), + values.length, + valueBytes.length, + OptionalLong.empty(), + RLE, + RLE, + PLAIN, + 0); + } + + private DataPage createNullablePage(Integer... values) + throws IOException + { + ValuesWriter writer = getValuesWriter(PLAIN, INT32, OptionalInt.empty()); + boolean[] isNull = new boolean[values.length]; + for (int index = 0; index < values.length; index++) { + if (values[index] == null) { + isNull[index] = true; + } + else { + writer.writeInteger(values[index]); + } + } + return createNullableDataPage(V1, writer, OPTIONAL_FIELD, isNull); + } + + private static PageReader getPlainPageReaderMock(DataPage... pages) + { + return new PageReader(new ParquetDataSourceId("test"), UNCOMPRESSED, ImmutableList.copyOf(pages).iterator(), false, false, Optional.empty(), -1, -1); + } + + private static List intValues(Block block) + { + ImmutableList.Builder values = ImmutableList.builder(); + for (int position = 0; position < block.getPositionCount(); position++) { + values.add(INTEGER.getInt(block, position)); + } + return values.build(); + } + + private static List nullableIntValues(Block block) + { + List values = new ArrayList<>(block.getPositionCount()); + for (int position = 0; position < block.getPositionCount(); position++) { + values.add(block.isNull(position) ? null : INTEGER.getInt(block, position)); + } + return values; + } + private static PageReader getNullOnlyPageReaderMock() throws IOException { From ceb91e9e08ecd113246304ad26ade0393a175092 Mon Sep 17 00:00:00 2001 From: Dain Sundstrom Date: Sun, 12 Jul 2026 20:37:31 -0700 Subject: [PATCH 14/18] Skip unselected Parquet pages Selected reads still pay decompression and decoding costs for pages that contain no selected values, leaving much of the sparse-read benefit unrealized. Allow flat readers to identify and bypass complete unselected data pages with bounded lookahead. Before advancing the chunk stream, copy only the buffered page that still aliases source-owned input. This keeps ownership local to the page reader without cross-layer reference tracking. --- .../parquet/reader/AbstractColumnReader.java | 17 +- .../io/trino/parquet/reader/ColumnChunk.java | 14 + .../io/trino/parquet/reader/ColumnReader.java | 17 + .../parquet/reader/NestedColumnReader.java | 7 +- .../io/trino/parquet/reader/PageReader.java | 322 ++++++++++++++++-- .../trino/parquet/reader/ParquetReader.java | 6 +- .../parquet/reader/flat/FlatColumnReader.java | 168 ++++++++- .../reader/AbstractColumnReaderTest.java | 2 +- .../trino/parquet/reader/TestPageReader.java | 163 +++++++++ .../reader/flat/TestFlatColumnReader.java | 137 ++++++++ 10 files changed, 811 insertions(+), 42 deletions(-) diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/AbstractColumnReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/AbstractColumnReader.java index 18313d41c291..71b3a3c6556c 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/AbstractColumnReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/AbstractColumnReader.java @@ -84,12 +84,25 @@ public void setPageReader(PageReader pageReader, Optional row // For dictionary based encodings - https://github.com/apache/parquet-format/blob/master/Encodings.md if (dictionaryPage != null) { log.debug("field %s, readDictionaryPage %s", field, dictionaryPage); - dictionaryDecoder = dictionaryDecoderProvider.create(dictionaryPage, isNonNull()); - produceDictionaryBlock = shouldProduceDictionaryBlock(rowRanges); + try { + dictionaryDecoder = dictionaryDecoderProvider.create(dictionaryPage, isNonNull()); + produceDictionaryBlock = shouldProduceDictionaryBlock(rowRanges); + } + finally { + pageReader.releaseCurrentPage(); + } } this.rowRanges = createRowRangesIterator(rowRanges); } + @Override + public void close() + { + if (pageReader != null) { + pageReader.close(); + } + } + protected abstract boolean isNonNull(); protected boolean produceDictionaryBlock() diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ColumnChunk.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ColumnChunk.java index 2aac0cfcd49d..5db4db5444d5 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ColumnChunk.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ColumnChunk.java @@ -17,6 +17,7 @@ import java.util.OptionalLong; +import static com.google.common.base.Preconditions.checkArgument; import static java.util.Objects.requireNonNull; public class ColumnChunk @@ -24,6 +25,7 @@ public class ColumnChunk private final Block block; private final int[] definitionLevels; private final int[] repetitionLevels; + private final int maxBlockPositionCount; private OptionalLong maxBlockSize; public ColumnChunk(Block block, int[] definitionLevels, int[] repetitionLevels) @@ -32,11 +34,18 @@ public ColumnChunk(Block block, int[] definitionLevels, int[] repetitionLevels) } public ColumnChunk(Block block, int[] definitionLevels, int[] repetitionLevels, OptionalLong maxBlockSize) + { + this(block, definitionLevels, repetitionLevels, maxBlockSize, block.getPositionCount()); + } + + public ColumnChunk(Block block, int[] definitionLevels, int[] repetitionLevels, OptionalLong maxBlockSize, int maxBlockPositionCount) { this.block = requireNonNull(block, "block is null"); this.definitionLevels = requireNonNull(definitionLevels, "definitionLevels is null"); this.repetitionLevels = requireNonNull(repetitionLevels, "repetitionLevels is null"); this.maxBlockSize = maxBlockSize; + checkArgument(maxBlockPositionCount >= block.getPositionCount(), "maxBlockPositionCount is less than block position count"); + this.maxBlockPositionCount = maxBlockPositionCount; } public Block getBlock() @@ -61,4 +70,9 @@ public long getMaxBlockSize() } return maxBlockSize.orElseThrow(); } + + public int getMaxBlockPositionCount() + { + return maxBlockPositionCount; + } } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ColumnReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ColumnReader.java index e6a42de857a5..805462b2896d 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ColumnReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ColumnReader.java @@ -17,6 +17,8 @@ public interface ColumnReader { + void close(); + boolean hasPageReader(); void setPageReader(PageReader pageReader, Optional rowRanges); @@ -34,4 +36,19 @@ default ColumnChunk readPrimitive(int[] positions, int offset, int positionCount { throw new UnsupportedOperationException("Selected positions are not supported"); } + + default ColumnChunk readPrimitivePageFiltered(int[] positions, int offset, int positionCount) + { + throw new UnsupportedOperationException("Page-filtered positions are not supported"); + } + + default int getDataPageReadCount() + { + return 0; + } + + default long preparePageFilteredRead(int[] positions, int offset, int positionCount, long maxBufferedBytes) + { + return 0; + } } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/NestedColumnReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/NestedColumnReader.java index a569e41df141..47932572f574 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/NestedColumnReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/NestedColumnReader.java @@ -490,14 +490,9 @@ else if (page instanceof DataPageV2 dataPageV2) { remainingPageValueCount = pageValueCount; rowRanges.resetForNewPage(page.getFirstRowIndex()); - // For a compressed data page, the memory used by the decompressed values data needs to be accounted - // for separately as ParquetCompressionUtils#decompress allocates a new byte array for the decompressed result. - // For an uncompressed data page, we read directly from input Slices whose memory usage is already accounted - // for in AbstractParquetDataSource#ReferenceCountedReader. - int dataPageSizeInBytes = pageReader.arePagesCompressed() ? page.getUncompressedSize() : 0; long dictionarySizeInBytes = dictionaryDecoder == null ? 0 : dictionaryDecoder.getRetainedSizeInBytes(); long repetitionBufferSizeInBytes = sizeOf(repetitionBuffer); - memoryContext.setBytes(dataPageSizeInBytes + dictionarySizeInBytes + repetitionBufferSizeInBytes); + memoryContext.setBytes(dictionarySizeInBytes + repetitionBufferSizeInBytes + pageReader.getRetainedPageBytes()); return true; } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/PageReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/PageReader.java index a3e5fcda0bb1..37b6843721d3 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/PageReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/PageReader.java @@ -14,8 +14,7 @@ package io.trino.parquet.reader; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Iterators; -import com.google.common.collect.PeekingIterator; +import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; import io.trino.parquet.DataPage; import io.trino.parquet.DataPageV1; @@ -37,7 +36,9 @@ import org.apache.parquet.internal.column.columnindex.OffsetIndex; import java.io.IOException; +import java.util.ArrayDeque; import java.util.Iterator; +import java.util.List; import java.util.Optional; import static com.google.common.base.Preconditions.checkArgument; @@ -53,11 +54,18 @@ public final class PageReader private final CompressionCodec codec; private final boolean hasOnlyDictionaryEncodedPages; private final boolean hasNoNulls; - private final PeekingIterator compressedPages; + private final Iterator compressedPages; + private final ArrayDeque bufferedPages = new ArrayDeque<>(); private final Optional blockDecryptor; private boolean dictionaryAlreadyRead; private int dataPageReadCount; + private long bufferedPageBytes; + private long bufferedPageRetainedBytes; + private long currentPageRetainedBytes; + private boolean currentPageOwned; + @Nullable + private Page currentPage; @Nullable private byte[] dataPageAad; @Nullable @@ -114,7 +122,7 @@ public PageReader( { this.dataSourceId = requireNonNull(dataSourceId, "dataSourceId is null"); this.codec = codec; - this.compressedPages = Iterators.peekingIterator(compressedPages); + this.compressedPages = requireNonNull(compressedPages, "compressedPages is null"); this.hasOnlyDictionaryEncodedPages = hasOnlyDictionaryEncodedPages; this.hasNoNulls = hasNoNulls; this.blockDecryptor = decryptionContext.map(ColumnDecryptionContext::dataDecryptor); @@ -136,20 +144,21 @@ public boolean hasOnlyDictionaryEncodedPages() public DataPage readPage() { - if (!compressedPages.hasNext()) { + if (!hasNext()) { return null; } - Page compressedPage = compressedPages.next(); - checkState(compressedPage instanceof DataPage, "Found page %s instead of a DataPage", compressedPage); - dataPageReadCount++; + Page compressedPage = nextCompressedPage(); try { + checkState(compressedPage instanceof DataPage, "Found page %s instead of a DataPage", compressedPage); + dataPageReadCount++; if (blockDecryptor.isPresent()) { AesCipherUtils.quickUpdatePageAAD(dataPageAad, ((DataPage) compressedPage).getPageIndex()); } Slice slice = decryptSliceIfNeeded(compressedPage.getSlice(), dataPageAad); if (compressedPage instanceof DataPageV1 dataPageV1) { - return new DataPageV1( - !arePagesCompressed() ? slice : decompress(dataSourceId, codec, slice, dataPageV1.getUncompressedSize()), + PageData pageData = getPageData(slice, dataPageV1.getUncompressedSize(), arePagesCompressed()); + DataPage page = new DataPageV1( + pageData.data(), dataPageV1.getValueCount(), dataPageV1.getUncompressedSize(), dataPageV1.getFirstRowIndex(), @@ -157,31 +166,46 @@ public DataPage readPage() dataPageV1.getDefinitionLevelEncoding(), dataPageV1.getValueEncoding(), dataPageV1.getPageIndex()); + replaceCurrentPage(page, pageData.retainedBytes(), pageData.owned()); + return page; } DataPageV2 dataPageV2 = (DataPageV2) compressedPage; - if (!dataPageV2.isCompressed()) { + if (!dataPageV2.isCompressed() && blockDecryptor.isEmpty()) { return dataPageV2; } - int uncompressedSize = dataPageV2.getUncompressedSize() + int uncompressedDataSize = dataPageV2.getUncompressedSize() - dataPageV2.getDefinitionLevels().length() - dataPageV2.getRepetitionLevels().length(); - return new DataPageV2( + PageData pageData = getPageData(slice, uncompressedDataSize, dataPageV2.isCompressed()); + Slice repetitionLevels = copySlice(dataPageV2.getRepetitionLevels()); + Slice definitionLevels = copySlice(dataPageV2.getDefinitionLevels()); + DataPage page = new DataPageV2( dataPageV2.getRowCount(), dataPageV2.getNullCount(), dataPageV2.getValueCount(), - dataPageV2.getRepetitionLevels(), - dataPageV2.getDefinitionLevels(), + repetitionLevels, + definitionLevels, dataPageV2.getDataEncoding(), - decompress(dataSourceId, codec, slice, uncompressedSize), + pageData.data(), dataPageV2.getUncompressedSize(), dataPageV2.getFirstRowIndex(), dataPageV2.getStatistics(), false, dataPageV2.getPageIndex()); + replaceCurrentPage( + page, + repetitionLevels.length() + definitionLevels.length() + pageData.retainedBytes(), + true); + return page; } catch (IOException e) { + releaseCurrentPage(); throw new RuntimeException("Could not decompress page", e); } + catch (RuntimeException | Error e) { + releaseCurrentPage(); + throw e; + } } public DictionaryPage readDictionaryPage() @@ -189,41 +213,291 @@ public DictionaryPage readDictionaryPage() checkState(!dictionaryAlreadyRead, "Dictionary was already read"); checkState(dataPageReadCount == 0, "Dictionary has to be read first but %s was read already", dataPageReadCount); dictionaryAlreadyRead = true; - if (!(compressedPages.peek() instanceof DictionaryPage)) { + if (!(peekCompressedPage() instanceof DictionaryPage)) { return null; } try { - DictionaryPage compressedDictionaryPage = (DictionaryPage) compressedPages.next(); + DictionaryPage compressedDictionaryPage = (DictionaryPage) nextCompressedPage(); Slice slice = decryptSliceIfNeeded(compressedDictionaryPage.getSlice(), dictionaryPageAad); - return new DictionaryPage( - decompress(dataSourceId, codec, slice, compressedDictionaryPage.getUncompressedSize()), + PageData pageData = getPageData(slice, compressedDictionaryPage.getUncompressedSize(), arePagesCompressed()); + DictionaryPage dictionaryPage = new DictionaryPage( + pageData.data(), compressedDictionaryPage.getDictionarySize(), compressedDictionaryPage.getEncoding()); + replaceCurrentPage(dictionaryPage, pageData.retainedBytes(), pageData.owned()); + return dictionaryPage; } catch (IOException e) { + releaseCurrentPage(); throw new RuntimeException("Error reading dictionary page", e); } + catch (RuntimeException | Error e) { + releaseCurrentPage(); + throw e; + } } public boolean hasNext() { - return compressedPages.hasNext(); + return !bufferedPages.isEmpty() || compressedPages.hasNext(); } public DataPage getNextPage() { verifyDictionaryPageRead(); - return (DataPage) compressedPages.peek(); + return (DataPage) peekCompressedPage(); } public void skipNextPage() { verifyDictionaryPageRead(); - compressedPages.next(); + nextCompressedPage(); + releaseCurrentPage(); + } + + public List getNextDataPages(int valueCount, int maxPageCount, long maxBufferedBytes) + { + verifyDictionaryPageRead(); + checkArgument(valueCount >= 0, "valueCount is negative"); + checkArgument(maxPageCount > 0, "maxPageCount must be positive"); + checkArgument(maxBufferedBytes > 0, "maxBufferedBytes must be positive"); + + int bufferedValueCount = 0; + int pageCount = 0; + for (BufferedPage bufferedPage : bufferedPages) { + Page page = bufferedPage.page(); + checkState(page instanceof DataPage, "Found page %s instead of a DataPage", page); + DataPage dataPage = (DataPage) page; + pageCount++; + bufferedValueCount += dataPage.getValueCount(); + if (bufferedValueCount >= valueCount || pageCount >= maxPageCount) { + return getBufferedDataPages(valueCount, maxPageCount); + } + } + + while (bufferedValueCount < valueCount && pageCount < maxPageCount && bufferedPageBytes < maxBufferedBytes && canAdvanceInput() && compressedPages.hasNext()) { + ownLastBufferedPage(); + Page page = compressedPages.next(); + checkState(page instanceof DataPage, "Found page %s instead of a DataPage", page); + addBufferedPage(page); + DataPage dataPage = (DataPage) page; + bufferedValueCount += dataPage.getValueCount(); + pageCount++; + } + return getBufferedDataPages(valueCount, maxPageCount); + } + + public long getRetainedPageBytes() + { + return bufferedPageRetainedBytes + currentPageRetainedBytes; + } + + @VisibleForTesting + public int getDataPageReadCount() + { + return dataPageReadCount; + } + + private Page peekCompressedPage() + { + if (bufferedPages.isEmpty()) { + addBufferedPage(compressedPages.next()); + } + return bufferedPages.peekFirst().page(); + } + + private Page nextCompressedPage() + { + releaseCurrentPage(); + if (!bufferedPages.isEmpty()) { + BufferedPage bufferedPage = bufferedPages.removeFirst(); + bufferedPageBytes -= getPageBytes(bufferedPage.page()); + bufferedPageRetainedBytes -= bufferedPage.retainedBytes(); + currentPage = bufferedPage.page(); + currentPageRetainedBytes = bufferedPage.retainedBytes(); + currentPageOwned = bufferedPage.owned(); + return currentPage; + } + currentPage = compressedPages.next(); + currentPageOwned = false; + return currentPage; + } + + public void releaseCurrentPage() + { + currentPage = null; + currentPageRetainedBytes = 0; + currentPageOwned = false; + } + + public void close() + { + releaseCurrentPage(); + bufferedPages.clear(); + bufferedPageBytes = 0; + bufferedPageRetainedBytes = 0; + } + + private void replaceCurrentPage(Page page, long retainedBytes, boolean owned) + { + currentPage = requireNonNull(page, "page is null"); + currentPageRetainedBytes = retainedBytes; + currentPageOwned = owned; + } + + private void addBufferedPage(Page page) + { + BufferedPage bufferedPage = new BufferedPage(page, 0, false); + bufferedPages.addLast(bufferedPage); + bufferedPageBytes += getPageBytes(page); + } + + private void ownLastBufferedPage() + { + if (!bufferedPages.isEmpty()) { + BufferedPage bufferedPage = bufferedPages.removeLast(); + if (!bufferedPage.owned()) { + BufferedPage ownedPage = copyPage(bufferedPage.page()); + bufferedPageRetainedBytes += ownedPage.retainedBytes(); + bufferedPage = ownedPage; + } + bufferedPages.addLast(bufferedPage); + } + } + + private boolean canAdvanceInput() + { + return currentPage == null || currentPageOwned; + } + + private List getBufferedDataPages(int valueCount, int maxPageCount) + { + ImmutableList.Builder pages = ImmutableList.builder(); + int bufferedValueCount = 0; + int pageCount = 0; + for (BufferedPage bufferedPage : bufferedPages) { + Page page = bufferedPage.page(); + checkState(page instanceof DataPage, "Found page %s instead of a DataPage", page); + DataPage dataPage = (DataPage) page; + pages.add(dataPage); + bufferedValueCount += dataPage.getValueCount(); + pageCount++; + if (bufferedValueCount >= valueCount || pageCount >= maxPageCount) { + break; + } + } + return pages.build(); + } + + private static Slice copySlice(Slice slice) + { + return wrappedBuffer(slice.getBytes()); + } + + private PageData getPageData(Slice data, int uncompressedSize, boolean compressed) + throws IOException + { + if (compressed) { + Slice uncompressed = arePagesCompressed() + ? decompress(dataSourceId, codec, data, uncompressedSize) + : copySlice(data); + return new PageData(uncompressed, uncompressed.length(), true); + } + if (blockDecryptor.isPresent()) { + return new PageData(data, data.length(), true); + } + return new PageData(data, currentPageRetainedBytes, currentPageOwned); + } + + private static BufferedPage copyPage(Page page) + { + if (page instanceof DataPageV1 dataPageV1) { + Slice data = copySlice(dataPageV1.getSlice()); + return new BufferedPage( + new DataPageV1( + data, + dataPageV1.getValueCount(), + dataPageV1.getUncompressedSize(), + dataPageV1.getFirstRowIndex(), + dataPageV1.getRepetitionLevelEncoding(), + dataPageV1.getDefinitionLevelEncoding(), + dataPageV1.getValueEncoding(), + dataPageV1.getPageIndex()), + data.length(), + true); + } + if (page instanceof DataPageV2 dataPageV2) { + int repetitionLevelsLength = dataPageV2.getRepetitionLevels().length(); + int definitionLevelsLength = dataPageV2.getDefinitionLevels().length(); + int dataLength = dataPageV2.getSlice().length(); + byte[] bytes = new byte[repetitionLevelsLength + definitionLevelsLength + dataLength]; + dataPageV2.getRepetitionLevels().getBytes(0, bytes, 0, repetitionLevelsLength); + dataPageV2.getDefinitionLevels().getBytes(0, bytes, repetitionLevelsLength, definitionLevelsLength); + dataPageV2.getSlice().getBytes(0, bytes, repetitionLevelsLength + definitionLevelsLength, dataLength); + Slice data = wrappedBuffer(bytes); + return new BufferedPage( + new DataPageV2( + dataPageV2.getRowCount(), + dataPageV2.getNullCount(), + dataPageV2.getValueCount(), + data.slice(0, repetitionLevelsLength), + data.slice(repetitionLevelsLength, definitionLevelsLength), + dataPageV2.getDataEncoding(), + data.slice(repetitionLevelsLength + definitionLevelsLength, dataLength), + dataPageV2.getUncompressedSize(), + dataPageV2.getFirstRowIndex(), + dataPageV2.getStatistics(), + dataPageV2.isCompressed(), + dataPageV2.getPageIndex()), + bytes.length, + true); + } + if (page instanceof DictionaryPage dictionaryPage) { + Slice data = copySlice(dictionaryPage.getSlice()); + return new BufferedPage( + new DictionaryPage( + data, + dictionaryPage.getUncompressedSize(), + dictionaryPage.getDictionarySize(), + dictionaryPage.getEncoding()), + data.length(), + true); + } + throw new IllegalArgumentException("Unsupported page: " + page); + } + + private static long getPageBytes(Page page) + { + if (page instanceof DataPageV2 dataPageV2) { + return (long) dataPageV2.getRepetitionLevels().length() + + dataPageV2.getDefinitionLevels().length() + + dataPageV2.getSlice().length(); + } + return page.getSlice().length(); + } + + private record BufferedPage(Page page, long retainedBytes, boolean owned) + { + private BufferedPage + { + requireNonNull(page, "page is null"); + checkArgument(retainedBytes >= 0, "retainedBytes is negative"); + checkArgument(owned || retainedBytes == 0, "borrowed page has retained bytes"); + } + } + + private record PageData(Slice data, long retainedBytes, boolean owned) + { + private PageData + { + requireNonNull(data, "data is null"); + checkArgument(retainedBytes >= 0, "retainedBytes is negative"); + checkArgument(owned || retainedBytes == 0, "borrowed data has retained bytes"); + } } - public boolean arePagesCompressed() + private boolean arePagesCompressed() { return codec != CompressionCodec.UNCOMPRESSED; } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java index fcaeb964f446..8f23f4fb172b 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java @@ -259,12 +259,13 @@ public void close() throws IOException { // Release memory usage from column readers + columnReaders.values().forEach(ColumnReader::close); columnReaders.clear(); - currentRowGroupMemoryContext.close(); for (ChunkedInputStream chunkedInputStream : chunkReaders.values()) { chunkedInputStream.close(); } + currentRowGroupMemoryContext.close(); dataSource.close(); if (writeChecksumBuilder.isPresent()) { @@ -501,9 +502,10 @@ private long[] selectedBatchRowNumbers() private boolean advanceToNextRowGroup() throws IOException { + columnReaders.values().forEach(ColumnReader::close); + freeCurrentRowGroupBuffers(); currentRowGroupMemoryContext.close(); currentRowGroupMemoryContext = memoryContext.newAggregatedMemoryContext(); - freeCurrentRowGroupBuffers(); if (currentRowGroup >= 0 && rowGroupStatisticsValidation.isPresent()) { StatisticsValidation statisticsValidation = rowGroupStatisticsValidation.get(); diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/flat/FlatColumnReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/flat/FlatColumnReader.java index 81bbe319472b..801bff58f4f8 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/flat/FlatColumnReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/flat/FlatColumnReader.java @@ -24,18 +24,24 @@ import io.trino.parquet.PrimitiveField; import io.trino.parquet.reader.AbstractColumnReader; import io.trino.parquet.reader.ColumnChunk; +import io.trino.parquet.reader.FilteredRowRanges; +import io.trino.parquet.reader.PageReader; import io.trino.parquet.reader.decoders.ValueDecoder; import io.trino.parquet.reader.decoders.ValueDecoder.ValueDecodersProvider; import io.trino.parquet.reader.flat.DictionaryDecoder.DictionaryDecoderProvider; import io.trino.parquet.reader.flat.FlatDefinitionLevelDecoder.DefinitionLevelDecoderProvider; +import io.trino.spi.block.Block; import io.trino.spi.block.RunLengthEncodedBlock; import io.trino.spi.type.Type; import java.util.Arrays; +import java.util.Optional; +import java.util.OptionalLong; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static io.trino.parquet.ParquetEncoding.RLE; +import static io.trino.parquet.reader.flat.RowRangesIterator.ALL_ROW_RANGES_ITERATOR; import static io.trino.spi.block.Bitmap.setBits; import static io.trino.spi.block.Bitmap.wordsForBits; import static java.lang.Math.min; @@ -50,15 +56,19 @@ public class FlatColumnReader private static final int[] EMPTY_DEFINITION_LEVELS = new int[0]; private static final int[] EMPTY_REPETITION_LEVELS = new int[0]; + private static final int MAX_PAGE_LOOKAHEAD = 8; private final DefinitionLevelDecoderProvider definitionLevelDecoderProvider; private final LocalMemoryContext memoryContext; private int remainingPageValueCount; + private int deferredPageValueCount; private FlatDefinitionLevelDecoder definitionLevelDecoder; private ValueDecoder valueDecoder; private int readOffset; private int nextBatchSize; + private final int[] skippedPageRanges = new int[MAX_PAGE_LOOKAHEAD * 2]; + private int skippedPageRangeCount; public FlatColumnReader( PrimitiveField field, @@ -79,6 +89,13 @@ public boolean hasPageReader() return pageReader != null; } + @Override + public void setPageReader(PageReader pageReader, Optional rowRanges) + { + super.setPageReader(pageReader, rowRanges); + updateMemoryUsage(); + } + @Override protected boolean isNonNull() { @@ -99,6 +116,8 @@ public ColumnChunk readPrimitive() readOffset = 0; nextBatchSize = 0; + skippedPageRangeCount = 0; + updateMemoryUsage(); return columnChunk; } @@ -108,6 +127,60 @@ public boolean supportsSelectedPositions() return true; } + @Override + public int getDataPageReadCount() + { + return pageReader == null ? 0 : pageReader.getDataPageReadCount(); + } + + @Override + public long preparePageFilteredRead(int[] positions, int offset, int positionCount, long maxBufferedBytes) + { + checkFromIndexSize(offset, positionCount, positions.length); + checkArgument(maxBufferedBytes > 0, "maxBufferedBytes must be positive"); + skippedPageRangeCount = 0; + if (pageReader == null || rowRanges != ALL_ROW_RANGES_ITERATOR) { + return 0; + } + if (remainingPageValueCount == 0) { + pageReader.releaseCurrentPage(); + updateMemoryUsage(); + } + + int valuesToConsume = readOffset + nextBatchSize; + int pageStart = remainingPageValueCount; + if (valuesToConsume <= pageStart) { + return 0; + } + + int selectedIndex = offset; + int selectedEnd = offset + positionCount; + long skippedPageBytes = 0; + boolean firstPage = true; + for (DataPage page : pageReader.getNextDataPages(valuesToConsume - pageStart + deferredPageValueCount, MAX_PAGE_LOOKAHEAD, maxBufferedBytes)) { + int pageValueCount = page.getValueCount() - (firstPage ? deferredPageValueCount : 0); + firstPage = false; + int pageEnd = pageStart + pageValueCount; + + while (selectedIndex < selectedEnd && readOffset + positions[selectedIndex] < pageStart) { + selectedIndex++; + } + boolean containsSelectedPosition = selectedIndex < selectedEnd && readOffset + positions[selectedIndex] < pageEnd; + if (pageStart >= readOffset && pageEnd <= valuesToConsume && !containsSelectedPosition) { + skippedPageBytes += page.getUncompressedSize(); + skippedPageRanges[skippedPageRangeCount * 2] = pageStart - readOffset; + skippedPageRanges[skippedPageRangeCount * 2 + 1] = pageEnd - readOffset; + skippedPageRangeCount++; + } + pageStart = pageEnd; + if (pageStart >= valuesToConsume) { + break; + } + } + updateMemoryUsage(); + return skippedPageBytes; + } + @Override public ColumnChunk readPrimitive(int[] positions, int offset, int positionCount) { @@ -124,9 +197,70 @@ public ColumnChunk readPrimitive(int[] positions, int offset, int positionCount) readOffset = 0; nextBatchSize = 0; + skippedPageRangeCount = 0; + updateMemoryUsage(); return columnChunk; } + @Override + public ColumnChunk readPrimitivePageFiltered(int[] positions, int offset, int positionCount) + { + checkFromIndexSize(offset, positionCount, positions.length); + checkState(skippedPageRangeCount > 0, "No skipped page ranges were identified"); + + int skippedPositionCount = 0; + for (int range = 0; range < skippedPageRangeCount; range++) { + skippedPositionCount += skippedPageRanges[range * 2 + 1] - skippedPageRanges[range * 2]; + } + int[] pagePositions = new int[nextBatchSize - skippedPositionCount]; + int pagePositionCount = 0; + int range = 0; + for (int position = 0; position < nextBatchSize; position++) { + while (range < skippedPageRangeCount && position >= skippedPageRanges[range * 2 + 1]) { + range++; + } + if (range >= skippedPageRangeCount || position < skippedPageRanges[range * 2]) { + pagePositions[pagePositionCount++] = position; + } + } + checkState(pagePositionCount == pagePositions.length, "Unexpected page position count"); + + seek(); + ColumnChunk pageChunk; + if (isNonNull()) { + pageChunk = readNonNull(pagePositions, 0, pagePositions.length); + } + else { + pageChunk = readNullable(pagePositions, 0, pagePositions.length); + } + + int[] resultPositions = new int[positionCount]; + int skippedBefore = 0; + range = 0; + for (int index = 0; index < positionCount; index++) { + int position = positions[offset + index]; + while (range < skippedPageRangeCount && skippedPageRanges[range * 2 + 1] <= position) { + skippedBefore += skippedPageRanges[range * 2 + 1] - skippedPageRanges[range * 2]; + range++; + } + checkState(range >= skippedPageRangeCount || position < skippedPageRanges[range * 2], "Selected position is in a skipped page"); + resultPositions[index] = position - skippedBefore; + } + Block block = pageChunk.getBlock().getPositions(resultPositions, 0, resultPositions.length); + ColumnChunk result = new ColumnChunk( + block, + pageChunk.getDefinitionLevels(), + pageChunk.getRepetitionLevels(), + OptionalLong.of(pageChunk.getMaxBlockSize()), + pagePositions.length); + + readOffset = 0; + nextBatchSize = 0; + skippedPageRangeCount = 0; + updateMemoryUsage(); + return result; + } + @Override public void prepareNextRead(int batchSize) { @@ -300,6 +434,7 @@ private void readSelectedPositions(int[] positions, int offset, int positionCoun while (positionOffset + runLength < endOffset && positions[positionOffset + runLength] == position + runLength) { runLength++; } + checkArgument(position + runLength <= nextBatchSize, "position %s is outside of batch size %s", position + runLength - 1, nextBatchSize); selectedPositionsReader.read(outputOffset, runLength); positionOffset += runLength - 1; batchPosition = position + runLength; @@ -346,6 +481,13 @@ private boolean readNextPage() } DataPage page = readPage(); rowRanges.resetForNewPage(page.getFirstRowIndex()); + if (deferredPageValueCount > 0) { + int skipCount = deferredPageValueCount; + deferredPageValueCount = 0; + int nonNullCount = isNonNull() ? skipCount : definitionLevelDecoder.skip(skipCount); + valueDecoder.skip(nonNullCount); + remainingPageValueCount -= skipCount; + } return true; } @@ -355,6 +497,18 @@ private int seekToNextPage(int remainingInBatch) { while (remainingInBatch > 0 && pageReader.hasNext()) { DataPage page = pageReader.getNextPage(); + if (rowRanges == ALL_ROW_RANGES_ITERATOR) { + int remainingPageValues = page.getValueCount() - deferredPageValueCount; + if (remainingInBatch < remainingPageValues) { + deferredPageValueCount += remainingInBatch; + return 0; + } + remainingInBatch -= remainingPageValues; + deferredPageValueCount = 0; + remainingPageValueCount = 0; + pageReader.skipNextPage(); + continue; + } rowRanges.resetForNewPage(page.getFirstRowIndex()); if (remainingInBatch < page.getValueCount() || !rowRanges.isPageFullyConsumed(page.getValueCount())) { readPage(); @@ -378,18 +532,18 @@ private DataPage readPage() else if (page instanceof DataPageV2 dataPageV2) { readFlatPageV2(dataPageV2); } - // For a compressed data page, the memory used by the decompressed values data needs to be accounted - // for separately as ParquetCompressionUtils#decompress allocates a new byte array for the decompressed result. - // For an uncompressed data page, we read directly from input Slices whose memory usage is already accounted - // for in AbstractParquetDataSource#ReferenceCountedReader. - int dataPageSizeInBytes = pageReader.arePagesCompressed() ? page.getUncompressedSize() : 0; - long dictionarySizeInBytes = dictionaryDecoder == null ? 0 : dictionaryDecoder.getRetainedSizeInBytes(); - memoryContext.setBytes(dataPageSizeInBytes + dictionarySizeInBytes); + updateMemoryUsage(); remainingPageValueCount = page.getValueCount(); return page; } + private void updateMemoryUsage() + { + long dictionarySizeInBytes = dictionaryDecoder == null ? 0 : dictionaryDecoder.getRetainedSizeInBytes(); + memoryContext.setBytes(dictionarySizeInBytes + pageReader.getRetainedPageBytes()); + } + private interface SelectedPositionsReader { void read(int outputOffset, int runLength); diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderTest.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderTest.java index aaf3cb86cc1e..d83af97afa90 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderTest.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderTest.java @@ -569,7 +569,7 @@ public void testMemoryUsage(DataPageVersion version, ColumnReaderFormat f // Read and assert assertThat(memoryContext.getBytes()).isEqualTo(0); reader.setPageReader(getPageReaderMock(List.of(page1, page2), dictionaryPage), Optional.empty()); - assertThat(memoryContext.getBytes()).isEqualTo(0); + assertThat(memoryContext.getBytes()).isGreaterThan(0); readBlock(reader, 3); long memoryUsage = memoryContext.getBytes(); assertThat(memoryUsage).isGreaterThan(0); diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestPageReader.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestPageReader.java index 538ea09ce25a..b1328cb46312 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestPageReader.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestPageReader.java @@ -26,10 +26,12 @@ import io.trino.parquet.ParquetDataSourceId; import io.trino.parquet.ParquetEncoding; import io.trino.parquet.ParquetTypeUtils; +import io.trino.parquet.crypto.ColumnDecryptionContext; import io.trino.parquet.metadata.ColumnChunkMetadata; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.EncodingStats; import org.apache.parquet.column.statistics.Statistics; +import org.apache.parquet.format.BlockCipher; import org.apache.parquet.format.CompressionCodec; import org.apache.parquet.format.DataPageHeader; import org.apache.parquet.format.DataPageHeaderV2; @@ -49,9 +51,12 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.OptionalLong; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -179,6 +184,138 @@ public void manyPages(CompressionCodec compressionCodec, DataPageType dataPageTy .hasMessageContaining("exceeds maximum allowed size"); } + @Test + public void testBufferedLookaheadCopiesAllButLastPage() + throws Exception + { + int valueCount = 10; + PageHeader pageHeader = new PageHeader(DATA_PAGE_V2, DATA_PAGE.length, DATA_PAGE.length); + V2.setDataPageHeader(pageHeader, valueCount); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (int page = 0; page < 3; page++) { + Util.writePageHeader(pageHeader, out); + out.write(DATA_PAGE); + } + Slice input = Slices.wrappedBuffer(out.toByteArray()); + PageReader pageReader = createPageReader(valueCount * 3, UNCOMPRESSED, false, ImmutableList.of(input)); + assertThat(pageReader.readDictionaryPage()).isNull(); + + List pages = pageReader.getNextDataPages(valueCount * 3, 8, Long.MAX_VALUE); + assertThat(pages).hasSize(3); + assertThat(pages.get(0).getSlice().byteArray()).isNotSameAs(input.byteArray()); + assertThat(pages.get(1).getSlice().byteArray()).isNotSameAs(input.byteArray()); + assertThat(pages.get(2).getSlice().byteArray()).isSameAs(input.byteArray()); + assertThat(pageReader.getRetainedPageBytes()).isEqualTo(DATA_PAGE.length * 2L); + + pageReader.skipNextPage(); + assertThat(pageReader.getRetainedPageBytes()).isEqualTo(DATA_PAGE.length); + pageReader.skipNextPage(); + assertThat(pageReader.getRetainedPageBytes()).isZero(); + pageReader.skipNextPage(); + assertThat(pageReader.getRetainedPageBytes()).isZero(); + } + + @Test + public void testEncryptedBufferedLookaheadOwnsDecryptedPages() + throws Exception + { + int valueCount = 10; + Slice input = Slices.wrappedBuffer(new byte[DATA_PAGE.length * 3]); + List encryptedPages = Stream.of(0, 1, 2) + .map(pageIndex -> new DataPageV1( + input.slice(pageIndex * DATA_PAGE.length, DATA_PAGE.length), + valueCount, + DATA_PAGE.length, + OptionalLong.empty(), + ParquetEncoding.RLE, + ParquetEncoding.RLE, + ParquetEncoding.PLAIN, + pageIndex)) + .collect(toImmutableList()); + BlockCipher.Decryptor decryptor = new CopyingDecryptor(); + PageReader pageReader = new PageReader( + new ParquetDataSourceId("test"), + UNCOMPRESSED, + encryptedPages.iterator(), + false, + false, + Optional.of(new ColumnDecryptionContext(decryptor, decryptor, new byte[0])), + 0, + 0); + assertThat(pageReader.readDictionaryPage()).isNull(); + + List bufferedPages = pageReader.getNextDataPages(valueCount * 3, 8, Long.MAX_VALUE); + assertThat(bufferedPages).hasSize(3); + assertThat(bufferedPages.get(0).getSlice().byteArray()).isNotSameAs(input.byteArray()); + assertThat(bufferedPages.get(1).getSlice().byteArray()).isNotSameAs(input.byteArray()); + assertThat(bufferedPages.get(2).getSlice().byteArray()).isSameAs(input.byteArray()); + + for (int pageIndex = 0; pageIndex < 3; pageIndex++) { + DataPage page = pageReader.readPage(); + assertThat(page.getSlice().byteArray()).isNotSameAs(input.byteArray()); + long expectedRetainedBytes = pageIndex == 0 ? DATA_PAGE.length * 2L : DATA_PAGE.length; + assertThat(pageReader.getRetainedPageBytes()).isEqualTo(expectedRetainedBytes); + } + pageReader.releaseCurrentPage(); + assertThat(pageReader.getRetainedPageBytes()).isZero(); + } + + @Test + public void testCompressedV2OwnsDecoderInput() + throws Exception + { + int valueCount = 10; + byte[] compressedDataPage = V2.compress(SNAPPY, DATA_PAGE); + PageHeader pageHeader = new PageHeader(DATA_PAGE_V2, DATA_PAGE.length, compressedDataPage.length); + V2.setDataPageHeader(pageHeader, valueCount); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Util.writePageHeader(pageHeader, out); + out.write(compressedDataPage); + Slice input = Slices.wrappedBuffer(out.toByteArray()); + PageReader pageReader = createPageReader(valueCount, SNAPPY, false, ImmutableList.of(input)); + assertThat(pageReader.readDictionaryPage()).isNull(); + + DataPageV2 page = (DataPageV2) pageReader.readPage(); + assertThat(page.getRepetitionLevels().byteArray()).isNotSameAs(input.byteArray()); + assertThat(page.getDefinitionLevels().byteArray()).isNotSameAs(input.byteArray()); + assertThat(page.getSlice().byteArray()).isNotSameAs(input.byteArray()); + assertThat(pageReader.getRetainedPageBytes()).isEqualTo(DATA_PAGE.length); + + pageReader.releaseCurrentPage(); + assertThat(pageReader.getRetainedPageBytes()).isZero(); + } + + @Test + public void testUncompressedV2BorrowsDecoderInput() + throws Exception + { + int valueCount = 10; + PageHeader pageHeader = new PageHeader(DATA_PAGE_V2, DATA_PAGE.length, DATA_PAGE.length); + V2.setDataPageHeader(pageHeader, valueCount); + pageHeader.getData_page_header_v2().setIs_compressed(false); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Util.writePageHeader(pageHeader, out); + out.write(DATA_PAGE); + Util.writePageHeader(pageHeader, out); + out.write(DATA_PAGE); + Slice input = Slices.wrappedBuffer(out.toByteArray()); + PageReader pageReader = createPageReader(valueCount * 2, SNAPPY, false, ImmutableList.of(input)); + assertThat(pageReader.readDictionaryPage()).isNull(); + + DataPageV2 page = (DataPageV2) pageReader.readPage(); + assertThat(page.getRepetitionLevels().byteArray()).isSameAs(input.byteArray()); + assertThat(page.getDefinitionLevels().byteArray()).isSameAs(input.byteArray()); + assertThat(page.getSlice().byteArray()).isSameAs(input.byteArray()); + assertThat(pageReader.getRetainedPageBytes()).isZero(); + assertThat(pageReader.getNextDataPages(valueCount, 8, Long.MAX_VALUE)).isEmpty(); + + pageReader.releaseCurrentPage(); + assertThat(pageReader.getNextDataPages(valueCount, 8, Long.MAX_VALUE)).hasSize(1); + } + @ParameterizedTest @MethodSource("pageParameters") public void dictionaryPage(CompressionCodec compressionCodec, DataPageType dataPageType) @@ -424,6 +561,32 @@ private static byte[] compress(CompressionCodec compressionCodec, byte[] bytes, throw new IllegalArgumentException("unsupported compression code " + compressionCodec); } + private static class CopyingDecryptor + implements BlockCipher.Decryptor + { + @Override + public byte[] decrypt(byte[] ciphertext, byte[] aad) + { + return ciphertext.clone(); + } + + @Override + public ByteBuffer decrypt(ByteBuffer ciphertext, byte[] aad) + { + ByteBuffer input = ciphertext.duplicate(); + byte[] plaintext = new byte[input.remaining()]; + input.get(plaintext); + return ByteBuffer.wrap(plaintext); + } + + @Override + public byte[] decrypt(InputStream ciphertext, byte[] aad) + throws IOException + { + return ciphertext.readAllBytes(); + } + } + private static PageReader createPageReader(int valueCount, CompressionCodec compressionCodec, boolean hasDictionary, List slices) { return createPageReader(valueCount, compressionCodec, hasDictionary, slices, Long.MAX_VALUE); diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/flat/TestFlatColumnReader.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/flat/TestFlatColumnReader.java index d248a31c393a..0b14b42f9943 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/flat/TestFlatColumnReader.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/flat/TestFlatColumnReader.java @@ -23,6 +23,7 @@ import io.trino.parquet.ParquetReaderOptions; import io.trino.parquet.PrimitiveField; import io.trino.parquet.reader.AbstractColumnReaderTest; +import io.trino.parquet.reader.ColumnChunk; import io.trino.parquet.reader.ColumnReader; import io.trino.parquet.reader.ColumnReaderFactory; import io.trino.parquet.reader.PageReader; @@ -243,6 +244,142 @@ public void testReadEmptySelection() assertThat(intValues(reader.readPrimitive().getBlock())).containsExactly(10, 11); } + @Test + public void testEmptySelectionsDeferCompressedPageAcrossBatches() + throws IOException + { + FlatColumnReader reader = (FlatColumnReader) createColumnReader(FIELD); + PageReader pageReader = getPlainPageReaderMock( + createPlainDataPage(0, 1, 2, 3, 4, 5, 6, 7, 8, 9), + createPlainDataPage(10, 11)); + reader.setPageReader(pageReader, Optional.empty()); + + reader.prepareNextRead(4); + assertThat(reader.readPrimitive(new int[0], 0, 0).getBlock().getPositionCount()).isZero(); + assertThat(pageReader.getDataPageReadCount()).isZero(); + + reader.prepareNextRead(6); + assertThat(reader.readPrimitive(new int[0], 0, 0).getBlock().getPositionCount()).isZero(); + assertThat(pageReader.getDataPageReadCount()).isZero(); + + reader.prepareNextRead(2); + assertThat(intValues(reader.readPrimitive().getBlock())).containsExactly(10, 11); + assertThat(pageReader.getDataPageReadCount()).isOne(); + } + + @Test + public void testSelectedPositionOpensDeferredCompressedPage() + throws IOException + { + FlatColumnReader reader = (FlatColumnReader) createColumnReader(FIELD); + PageReader pageReader = getPlainPageReaderMock( + createPlainDataPage(0, 1, 2, 3, 4, 5, 6, 7, 8, 9), + createPlainDataPage(10, 11)); + reader.setPageReader(pageReader, Optional.empty()); + + reader.prepareNextRead(4); + assertThat(reader.readPrimitive(new int[0], 0, 0).getBlock().getPositionCount()).isZero(); + assertThat(pageReader.getDataPageReadCount()).isZero(); + + reader.prepareNextRead(4); + assertThat(intValues(reader.readPrimitive(new int[] {1}, 0, 1).getBlock())).containsExactly(5); + assertThat(pageReader.getDataPageReadCount()).isOne(); + + reader.prepareNextRead(4); + assertThat(intValues(reader.readPrimitive().getBlock())).containsExactly(8, 9, 10, 11); + assertThat(pageReader.getDataPageReadCount()).isEqualTo(2); + } + + @Test + public void testNullableSelectedPositionsOpenDeferredCompressedPage() + throws IOException + { + FlatColumnReader reader = (FlatColumnReader) createColumnReader(OPTIONAL_FIELD); + PageReader pageReader = getPlainPageReaderMock( + createNullablePage(0, null, 2, 3, 4, null, 6, 7, 8, 9), + createNullablePage(10, 11)); + reader.setPageReader(pageReader, Optional.empty()); + + reader.prepareNextRead(4); + assertThat(reader.readPrimitive(new int[0], 0, 0).getBlock().getPositionCount()).isZero(); + assertThat(pageReader.getDataPageReadCount()).isZero(); + + reader.prepareNextRead(4); + assertThat(nullableIntValues(reader.readPrimitive(new int[] {1, 2}, 0, 2).getBlock())).containsExactly(null, 6); + assertThat(pageReader.getDataPageReadCount()).isOne(); + } + + @Test + public void testReadSelectedPositionsSkipsDataPages() + throws IOException + { + FlatColumnReader reader = (FlatColumnReader) createColumnReader(FIELD); + PageReader pageReader = getPlainPageReaderMock( + createPlainDataPage(0, 1, 2, 3, 4), + createPlainDataPage(5, 6, 7, 8, 9), + createPlainDataPage(10, 11, 12, 13, 14), + createPlainDataPage(15, 16, 17, 18, 19)); + reader.setPageReader(pageReader, Optional.empty()); + + reader.prepareNextRead(20); + assertThat(reader.preparePageFilteredRead(new int[] {11, 12}, 0, 2, Long.MAX_VALUE)).isPositive(); + ColumnChunk selectedChunk = reader.readPrimitivePageFiltered(new int[] {11, 12}, 0, 2); + Block selected = selectedChunk.getBlock(); + + assertThat(intValues(selected)).containsExactly(11, 12); + assertThat(selectedChunk.getMaxBlockSize()).isGreaterThan(selected.getSizeInBytes()); + assertThat(pageReader.getDataPageReadCount()).isOne(); + assertThat(pageReader.hasNext()).isFalse(); + } + + @Test + public void testPageLookaheadIsByteBounded() + throws IOException + { + FlatColumnReader reader = (FlatColumnReader) createColumnReader(FIELD); + PageReader pageReader = getPlainPageReaderMock( + createPlainDataPage(0, 1, 2, 3, 4), + createPlainDataPage(5, 6, 7, 8, 9), + createPlainDataPage(10, 11, 12, 13, 14)); + reader.setPageReader(pageReader, Optional.empty()); + + reader.prepareNextRead(15); + assertThat(reader.preparePageFilteredRead(new int[] {11}, 0, 1, 1)).isEqualTo(5 * Integer.BYTES); + } + + @Test + public void testReadNullablePositionsWithPageFiltering() + throws IOException + { + FlatColumnReader reader = (FlatColumnReader) createColumnReader(OPTIONAL_FIELD); + PageReader pageReader = getPlainPageReaderMock( + createNullablePage(0, null, 2, 3, null), + createNullablePage(5, null, 7, null, 9), + createNullablePage(10, 11, null, 13, 14)); + reader.setPageReader(pageReader, Optional.empty()); + + reader.prepareNextRead(15); + int[] positions = {1, 11}; + assertThat(reader.preparePageFilteredRead(positions, 0, positions.length, Long.MAX_VALUE)).isPositive(); + Block selected = reader.readPrimitivePageFiltered(positions, 0, positions.length).getBlock(); + + assertThat(nullableIntValues(selected)).containsExactly(null, 11); + assertThat(pageReader.getDataPageReadCount()).isEqualTo(2); + } + + @Test + public void testSelectedRunOutsideBatch() + throws IOException + { + FlatColumnReader reader = (FlatColumnReader) createColumnReader(FIELD); + reader.setPageReader(getPlainPageReaderMock(createPlainDataPage(0, 1, 2, 3)), Optional.empty()); + reader.prepareNextRead(4); + + assertThatThrownBy(() -> reader.readPrimitive(new int[] {3, 4}, 0, 2)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("position 4 is outside of batch size 4"); + } + @Test public void testReadSelectedNullablePositions() throws IOException From c114f8aff4c7d956c6a18d87a623a518c2e47af0 Mon Sep 17 00:00:00 2001 From: Dain Sundstrom Date: Sun, 12 Jul 2026 20:38:11 -0700 Subject: [PATCH 15/18] Select Parquet reads adaptively Selected decoding adds seek and run-processing overhead, so applying it to every filtered page can regress dense or fragmented selections. Masking a page after choosing a full read can also change downstream block representation without avoiding any reader work. Accept an engine selection only when at least one unloaded column will use row- or page-level skipping. Choose those modes only when encoding, physical type, null statistics, selectivity, run geometry, and confirmed skipped page bytes predict a meaningful gain. Otherwise leave the source page unchanged so projections retain the existing compact path. --- .../sphinx/object-storage/file-formats.md | 5 + .../trino/parquet/ParquetReaderOptions.java | 18 + .../trino/parquet/reader/ParquetReader.java | 642 +++++++++++++++++- .../io/trino/parquet/ParquetTestUtils.java | 17 +- .../parquet/reader/TestParquetReader.java | 548 +++++++++++++++ .../hive/parquet/ParquetReaderConfig.java | 15 + .../plugin/hive/TestParquetPageSkipping.java | 43 ++ .../hive/parquet/TestParquetReaderConfig.java | 3 + .../plugin/redshift/TestRedshiftUnload.java | 2 +- 9 files changed, 1262 insertions(+), 31 deletions(-) diff --git a/docs/src/main/sphinx/object-storage/file-formats.md b/docs/src/main/sphinx/object-storage/file-formats.md index 2bdb9e1e1435..2879a544b58f 100644 --- a/docs/src/main/sphinx/object-storage/file-formats.md +++ b/docs/src/main/sphinx/object-storage/file-formats.md @@ -111,6 +111,11 @@ with Parquet files performed by supported object storage connectors: entirely. The equivalent catalog session property is named `parquet_small_file_threshold`. - `3MB` +* - `parquet.selected-positions-pushdown-enabled` + - Allow the Parquet reader to use rows selected by query filters to skip + decoding rejected values and decompressing entirely unselected data pages + when beneficial. Set to `false` to disable filter selection pushdown. + - `true` * - `parquet.experimental.vectorized-decoding.enabled` - Enable using Java Vector API (SIMD) for faster decoding of parquet files. The equivalent catalog session property is diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/ParquetReaderOptions.java b/lib/trino-parquet/src/main/java/io/trino/parquet/ParquetReaderOptions.java index 4ca8e2002540..9972e548fefc 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/ParquetReaderOptions.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/ParquetReaderOptions.java @@ -39,6 +39,7 @@ public class ParquetReaderOptions private final boolean useColumnIndex; private final boolean useBloomFilter; private final DataSize smallFileThreshold; + private final boolean selectedPositionsPushdownEnabled; private final boolean vectorizedDecodingEnabled; private final DataSize footerReadSize; private final DataSize maxFooterReadSize; @@ -54,6 +55,7 @@ private ParquetReaderOptions() useColumnIndex = true; useBloomFilter = true; smallFileThreshold = DEFAULT_SMALL_FILE_THRESHOLD; + selectedPositionsPushdownEnabled = true; vectorizedDecodingEnabled = true; footerReadSize = DEFAULT_FOOTER_READ_SIZE; maxFooterReadSize = DEFAULT_MAX_FOOTER_READ_SIZE; @@ -69,6 +71,7 @@ private ParquetReaderOptions( boolean useColumnIndex, boolean useBloomFilter, DataSize smallFileThreshold, + boolean selectedPositionsPushdownEnabled, boolean vectorizedDecodingEnabled, DataSize footerReadSize, DataSize maxFooterReadSize, @@ -83,6 +86,7 @@ private ParquetReaderOptions( this.useColumnIndex = useColumnIndex; this.useBloomFilter = useBloomFilter; this.smallFileThreshold = requireNonNull(smallFileThreshold, "smallFileThreshold is null"); + this.selectedPositionsPushdownEnabled = selectedPositionsPushdownEnabled; this.vectorizedDecodingEnabled = vectorizedDecodingEnabled; this.footerReadSize = requireNonNull(footerReadSize, "footerReadSize is null"); checkArgument(footerReadSize.toBytes() >= 8, "footerReadSize must be at least 8 bytes"); @@ -150,6 +154,11 @@ public DataSize getSmallFileThreshold() return smallFileThreshold; } + public boolean isSelectedPositionsPushdownEnabled() + { + return selectedPositionsPushdownEnabled; + } + public DataSize getMaxFooterReadSize() { return maxFooterReadSize; @@ -175,6 +184,7 @@ public static class Builder private boolean useColumnIndex; private boolean useBloomFilter; private DataSize smallFileThreshold; + private boolean selectedPositionsPushdownEnabled; private boolean vectorizedDecodingEnabled; private DataSize footerReadSize; private DataSize maxFooterReadSize; @@ -191,6 +201,7 @@ private Builder(ParquetReaderOptions parquetReaderOptions) this.useColumnIndex = parquetReaderOptions.useColumnIndex; this.useBloomFilter = parquetReaderOptions.useBloomFilter; this.smallFileThreshold = parquetReaderOptions.smallFileThreshold; + this.selectedPositionsPushdownEnabled = parquetReaderOptions.selectedPositionsPushdownEnabled; this.vectorizedDecodingEnabled = parquetReaderOptions.vectorizedDecodingEnabled; this.footerReadSize = parquetReaderOptions.footerReadSize; this.maxFooterReadSize = parquetReaderOptions.maxFooterReadSize; @@ -245,6 +256,12 @@ public Builder withSmallFileThreshold(DataSize smallFileThreshold) return this; } + public Builder withSelectedPositionsPushdownEnabled(boolean selectedPositionsPushdownEnabled) + { + this.selectedPositionsPushdownEnabled = selectedPositionsPushdownEnabled; + return this; + } + public Builder withVectorizedDecodingEnabled(boolean vectorizedDecodingEnabled) { this.vectorizedDecodingEnabled = vectorizedDecodingEnabled; @@ -280,6 +297,7 @@ public ParquetReaderOptions build() useColumnIndex, useBloomFilter, smallFileThreshold, + selectedPositionsPushdownEnabled, vectorizedDecodingEnabled, footerReadSize, maxFooterReadSize, diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java index 8f23f4fb172b..5cbe3115e86b 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java @@ -13,6 +13,7 @@ */ package io.trino.parquet.reader; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -55,6 +56,7 @@ import io.trino.spi.metrics.Metric; import io.trino.spi.metrics.Metrics; import io.trino.spi.type.ArrayType; +import io.trino.spi.type.DecimalType; import io.trino.spi.type.MapType; import io.trino.spi.type.RowType; import jakarta.annotation.Nullable; @@ -65,6 +67,7 @@ import org.apache.parquet.internal.column.columnindex.OffsetIndex; import org.apache.parquet.internal.filter2.columnindex.ColumnIndexFilter; import org.apache.parquet.internal.filter2.columnindex.ColumnIndexStore; +import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; import org.joda.time.DateTimeZone; import java.io.Closeable; @@ -88,6 +91,7 @@ import static io.airlift.slice.SizeOf.instanceSize; import static io.airlift.slice.SizeOf.sizeOf; import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.parquet.ParquetReaderUtils.isOnlyDictionaryEncodingPages; import static io.trino.parquet.ParquetValidationUtils.validateParquet; import static io.trino.parquet.ParquetWriteValidation.StatisticsValidation.createStatisticsValidationBuilder; import static io.trino.parquet.ParquetWriteValidation.WriteChecksumBuilder.createWriteChecksumBuilder; @@ -101,8 +105,10 @@ import static java.lang.Math.min; import static java.lang.Math.toIntExact; import static java.lang.String.format; +import static java.util.Objects.checkFromIndexSize; import static java.util.Objects.checkIndex; import static java.util.Objects.requireNonNull; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; public class ParquetReader implements Closeable @@ -111,8 +117,19 @@ public class ParquetReader private static final int INITIAL_BATCH_SIZE = 1; private static final int BATCH_SIZE_GROWTH_FACTOR = 2; + private static final int MAX_FIXED_WIDTH_SELECTED_PERCENTAGE = 5; + private static final int MAX_NULLABLE_FIXED_WIDTH_SELECTED_PERCENTAGE = 20; + private static final int MAX_DICTIONARY_SELECTED_PERCENTAGE = 10; + private static final int MAX_BINARY_SELECTED_PERCENTAGE = 25; + private static final long VERY_WIDE_BINARY_BYTES = 1_024; + private static final int MAX_VERY_WIDE_BINARY_PROJECTED_COLUMNS = 4; + private static final int MIN_SKIPPED_POSITIONS_PER_RUN = 300; + private static final long MIN_SKIPPED_PAGE_BYTES = 4 * 1024; + private static final int MAX_PAGE_AWARE_SELECTED_PERCENTAGE = 75; + private static final int MAX_RETAINED_TO_LOGICAL_SIZE_RATIO = 2; public static final String PARQUET_CODEC_METRIC_PREFIX = "ParquetReaderCompressionFormat_"; public static final String COLUMN_INDEX_ROWS_FILTERED = "ParquetColumnIndexRowsFiltered"; + public static final String SELECTED_POSITIONS_PUSHDOWNS = "ParquetSelectedPositionsPushdowns"; private final Optional fileCreatedBy; private final List rowGroups; @@ -142,6 +159,7 @@ public class ParquetReader private int batchSize; private int nextBatchSize = INITIAL_BATCH_SIZE; private final Map columnReaders; + private final Map selectedPositionsPushdownCharacteristics = new HashMap<>(); private final Map maxBytesPerCell; private double maxCombinedBytesPerRow; private final ParquetReaderOptions options; @@ -157,9 +175,13 @@ public class ParquetReader private final Map> codecMetrics; private int currentPageId; + private int completedRowGroupDataPageReadCount; + private int selectedPositionsFallbackCount; + private int selectedPositionsPushdownCount; private long columnIndexRowsFiltered = -1; private final Optional decryptionContext; + private final boolean forceSelectedPositionsPushdown; public ParquetReader( Optional fileCreatedBy, @@ -175,6 +197,38 @@ public ParquetReader( Optional writeValidation, Optional decryptionContext) throws IOException + { + this(fileCreatedBy, + columnFields, + appendRowNumberColumn, + rowGroups, + dataSource, + timeZone, + memoryContext, + options, + exceptionTransform, + parquetPredicate, + writeValidation, + decryptionContext, + false); + } + + @VisibleForTesting + public ParquetReader( + Optional fileCreatedBy, + List columnFields, + boolean appendRowNumberColumn, + List rowGroups, + ParquetDataSource dataSource, + DateTimeZone timeZone, + AggregatedMemoryContext memoryContext, + ParquetReaderOptions options, + Function exceptionTransform, + Optional parquetPredicate, + Optional writeValidation, + Optional decryptionContext, + boolean forceSelectedPositionsPushdown) + throws IOException { this.fileCreatedBy = requireNonNull(fileCreatedBy, "fileCreatedBy is null"); requireNonNull(columnFields, "columnFields is null"); @@ -192,6 +246,7 @@ public ParquetReader( this.columnReaders = new HashMap<>(); this.maxBytesPerCell = new HashMap<>(); this.decryptionContext = requireNonNull(decryptionContext, "decryptionContext is null"); + this.forceSelectedPositionsPushdown = forceSelectedPositionsPushdown; this.writeValidation = requireNonNull(writeValidation, "writeValidation is null"); validateWrite( @@ -299,13 +354,17 @@ private class ParquetSourcePage private final long batchStartRow = lastBatchStartRow(); private final long[] batchRowNumbers = currentBatchRowNumbers; private SelectedPositions selectedPositions; + @Nullable + private SelectedPositionsReadMode[] selectedPositionsReadModes; + private int unloadedSelectedColumns; + private boolean selectedPositionsPushedDown; private long sizeInBytes; private long retainedSizeInBytes; public ParquetSourcePage(int positionCount) { - selectedPositions = new SelectedPositions(positionCount, null); + selectedPositions = SelectedPositions.allPositions(positionCount); retainedSizeInBytes = shallowRetainedSizeInBytes(); } @@ -332,6 +391,7 @@ private long shallowRetainedSizeInBytes() return INSTANCE_SIZE + sizeOf(blocks) + sizeOf(batchRowNumbers) + + sizeOf(selectedPositionsReadModes) + selectedPositions.retainedSizeInBytes(); } @@ -343,6 +403,9 @@ public void retainedBytesForEachPart(ObjLongConsumer consumer) if (batchRowNumbers != null) { consumer.accept(batchRowNumbers, sizeOf(batchRowNumbers)); } + if (selectedPositionsReadModes != null) { + consumer.accept(selectedPositionsReadModes, sizeOf(selectedPositionsReadModes)); + } consumer.accept(selectedPositions, selectedPositions.retainedSizeInBytes()); for (Block block : blocks) { if (block != null) { @@ -368,13 +431,20 @@ public Block getBlock(int channel) } else { try { - // todo use selected positions to improve read performance - block = readBlock(columnFields.get(channel).field()); + Field field = columnFields.get(channel).field(); + SelectedPositionsReadMode readMode = selectedPositionsReadModes == null + ? getSelectedPositionsReadMode(field, selectedPositions, unloadedSelectedColumns) + : requireNonNull(selectedPositionsReadModes[channel], "selected positions read mode is null"); + block = switch (readMode) { + case FULL -> compactIfRetainingExcessData(selectedPositions.apply(readBlock(field))); + case PAGE -> compactIfRetainingExcessData(readPrimitivePageFiltered((PrimitiveField) field, selectedPositions).getBlock()); + case ROW -> compactIfRetainingExcessData(readPrimitive((PrimitiveField) field, selectedPositions).getBlock()); + }; + recordSelectionDecision(readMode); } catch (IOException e) { throw exceptionTransform.apply(e); } - block = selectedPositions.apply(block); } blocks[channel] = block; sizeInBytes += block.getSizeInBytes(); @@ -393,31 +463,215 @@ public Page getPage() return new Page(selectedPositions.positionCount(), blocks); } + @Override + public boolean trySelectPositions(int[] positions, int offset, int size) + { + if (!options.isSelectedPositionsPushdownEnabled()) { + return false; + } + SelectedPositions newSelectedPositions = selectedPositions.selectPositionsView(positions, offset, size); + SelectedPositionsReadMode[] readModes = new SelectedPositionsReadMode[columnFields.size()]; + int unloadedColumnCount = countUnloadedColumns(); + boolean beneficial = false; + for (int channel = 0; channel < columnFields.size(); channel++) { + if (blocks[channel] == null) { + try { + readModes[channel] = getSelectedPositionsReadMode(columnFields.get(channel).field(), newSelectedPositions, unloadedColumnCount); + } + catch (IOException e) { + throw exceptionTransform.apply(e); + } + beneficial |= readModes[channel] != SelectedPositionsReadMode.FULL; + } + } + if (!beneficial) { + selectedPositionsReadModes = null; + return false; + } + + selectedPositionsReadModes = readModes; + SelectedPositions retainedSelectedPositions = selectedPositions.hasPositions() + ? newSelectedPositions + : newSelectedPositions.retainedCopy(); + selectPositions(retainedSelectedPositions, positions, offset, size); + resetSelectionTracking(); + return true; + } + @Override public void selectPositions(int[] positions, int offset, int size) { - selectedPositions = selectedPositions.selectPositions(positions, offset, size); + selectedPositionsReadModes = null; + SelectedPositions newSelectedPositions = selectedPositions.selectPositions(positions, offset, size); + selectPositions(newSelectedPositions, positions, offset, size); + resetSelectionTracking(); + } + + private void resetSelectionTracking() + { + unloadedSelectedColumns = countUnloadedColumns(); + selectedPositionsPushedDown = false; + if (unloadedSelectedColumns == 0) { + selectedPositionsFallbackCount++; + } + } + + private int countUnloadedColumns() + { + int unloadedColumnCount = 0; + for (int channel = 0; channel < columnFields.size(); channel++) { + if (blocks[channel] == null) { + unloadedColumnCount++; + } + } + return unloadedColumnCount; + } + + private void recordSelectionDecision(SelectedPositionsReadMode readMode) + { + if (unloadedSelectedColumns == 0) { + return; + } + unloadedSelectedColumns--; + if (readMode != SelectedPositionsReadMode.FULL && !selectedPositionsPushedDown) { + selectedPositionsPushedDown = true; + selectedPositionsPushdownCount++; + } + if (unloadedSelectedColumns == 0 && !selectedPositionsPushedDown) { + selectedPositionsFallbackCount++; + } + } + + private void selectPositions(SelectedPositions newSelectedPositions, int[] positions, int offset, int size) + { + selectedPositions = newSelectedPositions; + sizeInBytes = 0; retainedSizeInBytes = shallowRetainedSizeInBytes(); for (int i = 0; i < blocks.length; i++) { Block block = blocks[i]; if (block != null) { // loaded blocks already reflect the previous selection, so the incoming // positions apply to them directly - block = block.getPositions(positions, offset, size); + block = selectedPositionsReadModes == null + ? block.getPositions(positions, offset, size) + : block.copyPositions(positions, offset, size); + sizeInBytes += block.getSizeInBytes(); retainedSizeInBytes += block.getRetainedSizeInBytes(); blocks[i] = block; } } } + + private Block compactIfRetainingExcessData(Block block) + { + if (!selectedPositions.hasPositions()) { + return block; + } + long sizeInBytes = block.getSizeInBytes(); + // Selected reads can return dictionary views over a much larger decoded block. + if (sizeInBytes > 0 && block.getRetainedSizeInBytes() > sizeInBytes * MAX_RETAINED_TO_LOGICAL_SIZE_RATIO) { + if (block instanceof DictionaryBlock dictionaryBlock) { + return dictionaryBlock.compact(); + } + return block.copyRegion(0, block.getPositionCount()); + } + return block; + } } - private record SelectedPositions(int positionCount, @Nullable int[] positions) + private record SelectedPositions( + int originalPositionCount, + int positionCount, + @Nullable int[] positions, + int positionsOffset, + SelectionAnalysis analysis) { private static final long INSTANCE_SIZE = instanceSize(SelectedPositions.class); + private static final long ANALYSIS_INSTANCE_SIZE = instanceSize(SelectionAnalysis.class); + + private static SelectedPositions allPositions(int positionCount) + { + return new SelectedPositions(positionCount, positionCount, null, 0, SelectionAnalysis.analyzed(true, positionCount == 0 ? 0 : 1, 0)); + } public long retainedSizeInBytes() { - return INSTANCE_SIZE + sizeOf(positions); + return INSTANCE_SIZE + ANALYSIS_INSTANCE_SIZE + sizeOf(positions); + } + + public boolean hasPositions() + { + return positions != null; + } + + public boolean strictlyAscending() + { + analyze(); + return analysis.strictlyAscending; + } + + public boolean isSingleRunAtPageEdge() + { + return positionCount > 0 + && selectedRunCount() == 1 + && (positions[positionsOffset] == 0 || positions[positionsOffset + positionCount - 1] == originalPositionCount - 1); + } + + public int selectedRunCount() + { + analyze(); + return analysis.selectedRunCount; + } + + public int maxSkippedPositionCount() + { + analyze(); + return analysis.maxSkippedPositionCount; + } + + public boolean hasRowSelectionLocality() + { + if (analysis.analyzed) { + return meetsRowSelectionLocality( + originalPositionCount - positionCount, + analysis.selectedRunCount, + analysis.maxSkippedPositionCount); + } + + checkFromIndexSize(positionsOffset, positionCount, positions.length); + int skippedPositionCount = originalPositionCount - positionCount; + + // Each disjoint window with excess span contains at least one distinct run break. + int maximumBeneficialRunCount = skippedPositionCount / MIN_SKIPPED_POSITIONS_PER_RUN; + if (positionCount > 1 && maximumBeneficialRunCount < positionCount) { + int sampleStride = toIntExact((positionCount - 1L + maximumBeneficialRunCount) / (maximumBeneficialRunCount + 1)); + int minimumSelectedRunCount = 1; + for (int index = 0; index < positionCount - 1; index += sampleStride) { + int nextIndex = min(index + sampleStride, positionCount - 1); + if ((long) positions[positionsOffset + nextIndex] - positions[positionsOffset + index] > nextIndex - index + && ++minimumSelectedRunCount > maximumBeneficialRunCount) { + return false; + } + } + } + + analyze(); + return analysis.strictlyAscending + && meetsRowSelectionLocality(skippedPositionCount, analysis.selectedRunCount, analysis.maxSkippedPositionCount); + } + + public boolean mayHaveMinimumPotentialSkippedBytes(long estimatedUncompressedBytesPerValue, long minimumSkippedPageBytes) + { + if (positionCount == 0) { + return hasMinimumPotentialSkippedBytes(originalPositionCount, estimatedUncompressedBytesPerValue, minimumSkippedPageBytes); + } + int maximumPotentialSkippedPositions = max(positions[positionsOffset], originalPositionCount - positions[positionsOffset + positionCount - 1] - 1); + for (int index = 0; index < positionCount - 1; index += 64) { + int nextIndex = min(index + 64, positionCount - 1); + int skippedPositions = positions[positionsOffset + nextIndex] - positions[positionsOffset + index] - (nextIndex - index); + maximumPotentialSkippedPositions = max(maximumPotentialSkippedPositions, skippedPositions); + } + return hasMinimumPotentialSkippedBytes(maximumPotentialSkippedPositions, estimatedUncompressedBytesPerValue, minimumSkippedPageBytes); } @CheckReturnValue @@ -426,14 +680,14 @@ public Block apply(Block block) if (positions == null) { return block; } - return block.getPositions(positions, 0, positionCount); + return block.getPositions(positions, positionsOffset, positionCount); } public Block createRowNumberBlock(long batchStartRow, @Nullable long[] batchRowNumbers) { long[] rowNumbers = new long[positionCount]; for (int i = 0; i < positionCount; i++) { - int position = positions == null ? i : positions[i]; + int position = positions == null ? i : positions[positionsOffset + i]; if (batchRowNumbers == null) { rowNumbers[i] = batchStartRow + position; } @@ -447,19 +701,284 @@ public Block createRowNumberBlock(long batchStartRow, @Nullable long[] batchRowN @CheckReturnValue public SelectedPositions selectPositions(int[] positions, int offset, int size) { + checkFromIndexSize(offset, size, positions.length); + int[] newPositions = new int[size]; if (this.positions == null) { for (int i = 0; i < size; i++) { - checkIndex(offset + i, positionCount); + int selectedPosition = positions[offset + i]; + checkIndex(selectedPosition, positionCount); + newPositions[i] = selectedPosition; + } + } + else { + for (int i = 0; i < size; i++) { + int selectedPosition = positions[offset + i]; + checkIndex(selectedPosition, positionCount); + newPositions[i] = this.positions[positionsOffset + selectedPosition]; } - return new SelectedPositions(size, Arrays.copyOfRange(positions, offset, offset + size)); } - int[] newPositions = new int[size]; - for (int i = 0; i < size; i++) { - newPositions[i] = this.positions[positions[offset + i]]; + return create(originalPositionCount, newPositions, 0, size); + } + + public SelectedPositions selectPositionsView(int[] positions, int offset, int size) + { + if (this.positions != null) { + return selectPositions(positions, offset, size); } - return new SelectedPositions(size, newPositions); + checkFromIndexSize(offset, size, positions.length); + return new SelectedPositions(originalPositionCount, size, positions, offset, new SelectionAnalysis()); } + + public SelectedPositions retainedCopy() + { + analyze(); + int[] retainedPositions = Arrays.copyOfRange(positions, positionsOffset, positionsOffset + positionCount); + return new SelectedPositions(originalPositionCount, positionCount, retainedPositions, 0, analysis); + } + + private static SelectedPositions create(int originalPositionCount, int[] positions, int offset, int size) + { + return new SelectedPositions(originalPositionCount, size, positions, offset, new SelectionAnalysis()); + } + + private void analyze() + { + if (analysis.analyzed) { + return; + } + checkFromIndexSize(positionsOffset, positionCount, positions.length); + boolean strictlyAscending = true; + int selectedRunCount = positionCount == 0 ? 0 : 1; + int maxSkippedPositionCount = positionCount == 0 ? originalPositionCount : positions[positionsOffset]; + for (int i = 0; i < positionCount; i++) { + checkIndex(positions[positionsOffset + i], originalPositionCount); + if (i > 0) { + if (positions[positionsOffset + i] <= positions[positionsOffset + i - 1]) { + strictlyAscending = false; + } + else if (positions[positionsOffset + i] != positions[positionsOffset + i - 1] + 1) { + selectedRunCount++; + maxSkippedPositionCount = max(maxSkippedPositionCount, positions[positionsOffset + i] - positions[positionsOffset + i - 1] - 1); + } + } + } + if (positionCount > 0 && strictlyAscending) { + maxSkippedPositionCount = max(maxSkippedPositionCount, originalPositionCount - positions[positionsOffset + positionCount - 1] - 1); + } + analysis.set(strictlyAscending, selectedRunCount, maxSkippedPositionCount); + } + } + + private static final class SelectionAnalysis + { + private boolean analyzed; + private boolean strictlyAscending; + private int selectedRunCount; + private int maxSkippedPositionCount; + + private static SelectionAnalysis analyzed(boolean strictlyAscending, int selectedRunCount, int maxSkippedPositionCount) + { + SelectionAnalysis analysis = new SelectionAnalysis(); + analysis.set(strictlyAscending, selectedRunCount, maxSkippedPositionCount); + return analysis; + } + + private void set(boolean strictlyAscending, int selectedRunCount, int maxSkippedPositionCount) + { + this.analyzed = true; + this.strictlyAscending = strictlyAscending; + this.selectedRunCount = selectedRunCount; + this.maxSkippedPositionCount = maxSkippedPositionCount; + } + } + + private record SelectedPositionsPushdownCharacteristics( + PrimitiveTypeName primitiveType, + long estimatedUncompressedBytesPerValue, + boolean dictionaryEncoded, + boolean hasNulls) {} + + private enum SelectedPositionsReadMode + { + FULL, + PAGE, + ROW, + } + + private SelectedPositionsReadMode getSelectedPositionsReadMode(Field field, SelectedPositions selectedPositions, int unloadedColumnCount) + throws IOException + { + if (!selectedPositions.hasPositions()) { + return SelectedPositionsReadMode.FULL; + } + if (!(field instanceof PrimitiveField primitiveField)) { + return SelectedPositionsReadMode.FULL; + } + if (primitiveField.getDescriptor().getPath().length != 1 + || primitiveField.getRepetitionLevel() != 0 + || !columnReaders.get(primitiveField.getId()).supportsSelectedPositions()) { + return SelectedPositionsReadMode.FULL; + } + if (forceSelectedPositionsPushdown) { + return selectedPositions.strictlyAscending() ? SelectedPositionsReadMode.ROW : SelectedPositionsReadMode.FULL; + } + SelectedPositionsPushdownCharacteristics characteristics = selectedPositionsPushdownCharacteristics.get(primitiveField.getId()); + if (characteristics == null) { + ColumnChunkMetadata metadata = currentBlockMetadata.getColumnChunkMetaData(primitiveField.getDescriptor()); + long valueCount = metadata.getValueCount(); + boolean hasNulls = !primitiveField.isRequired() + && (metadata.getStatistics() == null + || !metadata.getStatistics().isNumNullsSet() + || metadata.getStatistics().getNumNulls() > 0); + characteristics = new SelectedPositionsPushdownCharacteristics( + primitiveField.getDescriptor().getPrimitiveType().getPrimitiveTypeName(), + valueCount == 0 ? 1 : max(1, metadata.getTotalUncompressedSize() / valueCount), + isOnlyDictionaryEncodingPages(metadata), + hasNulls); + selectedPositionsPushdownCharacteristics.put(primitiveField.getId(), characteristics); + } + boolean potentiallyRowSelectionBeneficial = isRowSelectionBeneficial( + batchSize, + selectedPositions.positionCount(), + 0, + 0, + characteristics.primitiveType(), + characteristics.dictionaryEncoded(), + characteristics.hasNulls(), + characteristics.estimatedUncompressedBytesPerValue(), + unloadedColumnCount); + boolean potentiallyPageBeneficial = isPageSelectionBeneficial(primitiveField, characteristics) + && selectedPercentageAtMost(batchSize, selectedPositions.positionCount(), MAX_PAGE_AWARE_SELECTED_PERCENTAGE) + && selectedPositions.mayHaveMinimumPotentialSkippedBytes(characteristics.estimatedUncompressedBytesPerValue(), MIN_SKIPPED_PAGE_BYTES); + if (!potentiallyRowSelectionBeneficial && !potentiallyPageBeneficial) { + return SelectedPositionsReadMode.FULL; + } + + if (!potentiallyPageBeneficial && !selectedPositions.hasRowSelectionLocality()) { + return SelectedPositionsReadMode.FULL; + } + if (!selectedPositions.strictlyAscending()) { + return SelectedPositionsReadMode.FULL; + } + + boolean rowSelectionBeneficial = isRowSelectionBeneficial( + batchSize, + selectedPositions.positionCount(), + selectedPositions.selectedRunCount(), + selectedPositions.maxSkippedPositionCount(), + characteristics.primitiveType(), + characteristics.dictionaryEncoded(), + characteristics.hasNulls(), + characteristics.estimatedUncompressedBytesPerValue(), + unloadedColumnCount); + if (rowSelectionBeneficial && !selectedPositions.isSingleRunAtPageEdge()) { + return SelectedPositionsReadMode.ROW; + } + + potentiallyPageBeneficial &= hasMinimumPotentialSkippedBytes( + selectedPositions.maxSkippedPositionCount(), + characteristics.estimatedUncompressedBytesPerValue(), + MIN_SKIPPED_PAGE_BYTES); + if (!potentiallyPageBeneficial) { + return SelectedPositionsReadMode.FULL; + } + + ColumnReader columnReader = initializeColumnReader(primitiveField); + long skippedPageBytes = columnReader.preparePageFilteredRead( + selectedPositions.positions(), + selectedPositions.positionsOffset(), + selectedPositions.positionCount(), + options.getMaxReadBlockSize().toBytes()); + return skippedPageBytes >= MIN_SKIPPED_PAGE_BYTES ? SelectedPositionsReadMode.PAGE : SelectedPositionsReadMode.FULL; + } + + private static boolean isPageSelectionBeneficial(PrimitiveField field, SelectedPositionsPushdownCharacteristics characteristics) + { + if (characteristics.dictionaryEncoded()) { + return false; + } + if (field.getType() instanceof DecimalType) { + return true; + } + return switch (characteristics.primitiveType()) { + case BOOLEAN, INT32, INT64, FLOAT -> false; + default -> true; + }; + } + + @VisibleForTesting + static boolean isRowSelectionBeneficial( + int batchSize, + int selectedPositionCount, + int selectedRunCount, + int maxSkippedPositionCount, + PrimitiveTypeName primitiveType, + boolean dictionaryEncoded, + boolean hasNulls, + long estimatedUncompressedBytesPerValue, + int projectedColumnCount) + { + checkArgument(batchSize >= 0, "batchSize is negative"); + checkArgument(selectedPositionCount >= 0 && selectedPositionCount <= batchSize, "selectedPositionCount is invalid"); + checkArgument(selectedRunCount >= 0 && selectedRunCount <= selectedPositionCount, "selectedRunCount is invalid"); + checkArgument(maxSkippedPositionCount >= 0 && maxSkippedPositionCount <= batchSize - selectedPositionCount, "maxSkippedPositionCount is invalid"); + requireNonNull(primitiveType, "primitiveType is null"); + checkArgument(estimatedUncompressedBytesPerValue > 0, "estimatedUncompressedBytesPerValue must be positive"); + checkArgument(projectedColumnCount > 0, "projectedColumnCount must be positive"); + + if (selectedPositionCount == 0) { + return true; + } + + int skippedPositionCount = batchSize - selectedPositionCount; + // A single large gap cannot amortize the seek/read overhead of the remaining runs. + if (!meetsRowSelectionLocality(skippedPositionCount, selectedRunCount, maxSkippedPositionCount)) { + return false; + } + // The generic seek/read path regresses packed Boolean values without a selection-aware decoder. + if (primitiveType == PrimitiveTypeName.BOOLEAN) { + return false; + } + if (dictionaryEncoded) { + return selectedPercentageAtMost(batchSize, selectedPositionCount, MAX_DICTIONARY_SELECTED_PERCENTAGE); + } + if (primitiveType == BINARY) { + if (estimatedUncompressedBytesPerValue >= VERY_WIDE_BINARY_BYTES + && projectedColumnCount > MAX_VERY_WIDE_BINARY_PROJECTED_COLUMNS) { + return false; + } + return selectedPercentageAtMost(batchSize, selectedPositionCount, MAX_BINARY_SELECTED_PERCENTAGE); + } + int maximumSelectedPercentage = hasNulls ? MAX_NULLABLE_FIXED_WIDTH_SELECTED_PERCENTAGE : MAX_FIXED_WIDTH_SELECTED_PERCENTAGE; + return selectedPercentageAtMost(batchSize, selectedPositionCount, maximumSelectedPercentage); + } + + private static boolean selectedPercentageAtMost(int batchSize, int selectedPositionCount, int percentage) + { + return selectedPositionCount <= ((long) batchSize * percentage + 50) / 100; + } + + private static boolean hasMinimumSkippedPositionsPerRun(int skippedPositionCount, int selectedRunCount, int minimumSkippedPositionsPerRun) + { + return (long) skippedPositionCount >= (long) selectedRunCount * minimumSkippedPositionsPerRun; + } + + private static boolean meetsRowSelectionLocality(int skippedPositionCount, int selectedRunCount, int maxSkippedPositionCount) + { + if (!hasMinimumSkippedPositionsPerRun(skippedPositionCount, selectedRunCount, MIN_SKIPPED_POSITIONS_PER_RUN)) { + return false; + } + return selectedRunCount <= 1 || hasMinimumSkippedPositionsPerRun( + skippedPositionCount - maxSkippedPositionCount, + selectedRunCount - 1, + MIN_SKIPPED_POSITIONS_PER_RUN); + } + + private static boolean hasMinimumPotentialSkippedBytes(int skippedPositionCount, long estimatedUncompressedBytesPerValue, long minimumSkippedPageBytes) + { + return skippedPositionCount > 0 + && estimatedUncompressedBytesPerValue >= (minimumSkippedPageBytes + skippedPositionCount - 1) / skippedPositionCount; } /** @@ -470,6 +989,26 @@ public long lastBatchStartRow() return firstRowIndexInGroup + nextRowInGroup - batchSize; } + @VisibleForTesting + public int getDataPageReadCount() + { + return completedRowGroupDataPageReadCount + columnReaders.values().stream() + .mapToInt(ColumnReader::getDataPageReadCount) + .sum(); + } + + @VisibleForTesting + public int getSelectedPositionsFallbackCount() + { + return selectedPositionsFallbackCount; + } + + @VisibleForTesting + public int getSelectedPositionsPushdownCount() + { + return selectedPositionsPushdownCount; + } + private int nextBatch() throws IOException { @@ -719,6 +1258,57 @@ private FilteredOffsetIndex getFilteredOffsetIndex(FilteredRowRanges rowRanges, private ColumnChunk readPrimitive(PrimitiveField field) throws IOException + { + return readPrimitive(field, null); + } + + private ColumnChunk readPrimitive(PrimitiveField field, @Nullable SelectedPositions selectedPositions) + throws IOException + { + ColumnReader columnReader = initializeColumnReader(field); + ColumnChunk columnChunk; + if (selectedPositions != null && selectedPositions.positions() != null) { + columnChunk = columnReader.readPrimitive(selectedPositions.positions(), selectedPositions.positionsOffset(), selectedPositions.positionCount()); + } + else { + columnChunk = columnReader.readPrimitive(); + } + + updateMaxBytesPerCell(field.getId(), columnChunk); + return columnChunk; + } + + private ColumnChunk readPrimitivePageFiltered(PrimitiveField field, SelectedPositions selectedPositions) + throws IOException + { + ColumnReader columnReader = initializeColumnReader(field); + ColumnChunk columnChunk = columnReader.readPrimitivePageFiltered( + selectedPositions.positions(), + selectedPositions.positionsOffset(), + selectedPositions.positionCount()); + updateMaxBytesPerCell(field.getId(), columnChunk); + return columnChunk; + } + + private void updateMaxBytesPerCell(int fieldId, ColumnChunk columnChunk) + { + if (columnChunk.getMaxBlockPositionCount() == 0) { + return; + } + double previousBytesPerCell = maxBytesPerCell.getOrDefault(fieldId, 0.0); + double bytesPerCell = max(previousBytesPerCell, ((double) columnChunk.getMaxBlockSize()) / columnChunk.getMaxBlockPositionCount()); + + if (bytesPerCell != previousBytesPerCell) { + maxBytesPerCell.put(fieldId, bytesPerCell); + maxCombinedBytesPerRow = max(0, maxCombinedBytesPerRow + bytesPerCell - previousBytesPerCell); + maxBatchSize = maxCombinedBytesPerRow == 0 + ? options.getMaxReadBlockRowCount() + : toIntExact(min(options.getMaxReadBlockRowCount(), max(1, (long) (options.getMaxReadBlockSize().toBytes() / maxCombinedBytesPerRow)))); + } + } + + private ColumnReader initializeColumnReader(PrimitiveField field) + throws IOException { ColumnDescriptor columnDescriptor = field.getDescriptor(); int fieldId = field.getId(); @@ -744,18 +1334,7 @@ private ColumnChunk readPrimitive(PrimitiveField field) options.getMaxPageReadSize().toBytes()), Optional.ofNullable(rowRanges)); } - ColumnChunk columnChunk = columnReader.readPrimitive(); - - // update max size per primitive column chunk - double bytesPerCell = ((double) columnChunk.getMaxBlockSize()) / batchSize; - double bytesPerCellDelta = bytesPerCell - maxBytesPerCell.getOrDefault(fieldId, 0.0); - if (bytesPerCellDelta > 0) { - // update batch size - maxCombinedBytesPerRow += bytesPerCellDelta; - maxBatchSize = toIntExact(min(maxBatchSize, max(1, (long) (options.getMaxReadBlockSize().toBytes() / maxCombinedBytesPerRow)))); - maxBytesPerCell.put(fieldId, bytesPerCell); - } - return columnChunk; + return columnReader; } public List getColumnFields() @@ -766,7 +1345,8 @@ public List getColumnFields() public Metrics getMetrics() { ImmutableMap.Builder> metrics = ImmutableMap.>builder() - .putAll(codecMetrics); + .putAll(codecMetrics) + .put(SELECTED_POSITIONS_PUSHDOWNS, new LongCount(selectedPositionsPushdownCount)); if (columnIndexRowsFiltered >= 0) { metrics.put(COLUMN_INDEX_ROWS_FILTERED, new LongCount(columnIndexRowsFiltered)); } @@ -777,6 +1357,10 @@ public Metrics getMetrics() private void initializeColumnReaders() { + completedRowGroupDataPageReadCount += columnReaders.values().stream() + .mapToInt(ColumnReader::getDataPageReadCount) + .sum(); + selectedPositionsPushdownCharacteristics.clear(); for (PrimitiveField field : primitiveFields) { columnReaders.put( field.getId(), diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/ParquetTestUtils.java b/lib/trino-parquet/src/test/java/io/trino/parquet/ParquetTestUtils.java index d9908c8c6c8a..5d8b5e623532 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/ParquetTestUtils.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/ParquetTestUtils.java @@ -138,6 +138,20 @@ public static ParquetReader createParquetReader( List columnNames, TupleDomain predicate) throws IOException + { + return createParquetReader(input, parquetMetadata, options, memoryContext, types, columnNames, predicate, false); + } + + public static ParquetReader createParquetReader( + ParquetDataSource input, + ParquetMetadata parquetMetadata, + ParquetReaderOptions options, + AggregatedMemoryContext memoryContext, + List types, + List columnNames, + TupleDomain predicate, + boolean forceSelectedPositionsPushdown) + throws IOException { FileMetadata fileMetaData = parquetMetadata.getFileMetaData(); MessageType fileSchema = fileMetaData.getSchema(); @@ -181,7 +195,8 @@ public static ParquetReader createParquetReader( }, Optional.of(parquetPredicate), Optional.empty(), - Optional.empty()); + Optional.empty(), + forceSelectedPositionsPushdown); } public static List generateInputPages(List types, int positionsPerPage, int pageCount) diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestParquetReader.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestParquetReader.java index 2334ff23bcaa..e184b8c67a06 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestParquetReader.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestParquetReader.java @@ -29,10 +29,12 @@ import io.trino.parquet.metadata.ParquetMetadata; import io.trino.parquet.predicate.TupleDomainParquetPredicate; import io.trino.parquet.writer.ParquetWriterOptions; +import io.trino.plugin.base.metrics.LongCount; import io.trino.spi.Page; import io.trino.spi.TrinoException; import io.trino.spi.block.Block; import io.trino.spi.block.BlockBuilder; +import io.trino.spi.block.DictionaryBlock; import io.trino.spi.connector.SourcePage; import io.trino.spi.metrics.Count; import io.trino.spi.metrics.Metric; @@ -41,6 +43,7 @@ import io.trino.spi.predicate.TupleDomain; import io.trino.spi.predicate.ValueSet; import io.trino.spi.type.ArrayType; +import io.trino.spi.type.BooleanType; import io.trino.spi.type.Type; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.io.MessageColumnIO; @@ -59,7 +62,9 @@ import java.util.Optional; import java.util.stream.IntStream; +import static io.airlift.slice.Slices.utf8Slice; import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static io.trino.parquet.ParquetTestUtils.createArrayBlock; import static io.trino.parquet.ParquetTestUtils.createParquetReader; import static io.trino.parquet.ParquetTestUtils.generateInputPages; import static io.trino.parquet.ParquetTestUtils.writeParquetFile; @@ -70,18 +75,105 @@ import static io.trino.parquet.predicate.PredicateUtils.buildPredicate; import static io.trino.parquet.predicate.PredicateUtils.getFilteredRowGroups; import static io.trino.parquet.reader.ParquetReader.COLUMN_INDEX_ROWS_FILTERED; +import static io.trino.parquet.reader.ParquetReader.SELECTED_POSITIONS_PUSHDOWNS; +import static io.trino.parquet.reader.ParquetReader.isRowSelectionBeneficial; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.spi.type.DateType.DATE; import static io.trino.spi.type.IntegerType.INTEGER; import static io.trino.spi.type.VarcharType.VARCHAR; import static java.lang.Math.min; import static java.lang.Math.toIntExact; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BINARY; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.BOOLEAN; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY; +import static org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName.INT32; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.joda.time.DateTimeZone.UTC; public class TestParquetReader { + @Test + public void testSelectedPositionsPushdownHeuristic() + { + assertThat(isRowSelectionBeneficial(8_192, 0, 0, 8_192, INT32, false, false, 4, 1)).isTrue(); + + // Required fixed-width values use a 5% selectivity limit. + assertThat(isRowSelectionBeneficial(8_192, 410, 16, 500, INT32, false, false, 4, 1)).isTrue(); + assertThat(isRowSelectionBeneficial(8_192, 411, 16, 500, INT32, false, false, 4, 1)).isFalse(); + + // Every value family uses the same locality requirement. + assertThat(isRowSelectionBeneficial(8_192, 409, 25, 300, INT32, false, false, 4, 1)).isTrue(); + assertThat(isRowSelectionBeneficial(8_192, 409, 26, 300, INT32, false, false, 4, 1)).isFalse(); + + // One large skipped range cannot make a fragmented remainder worthwhile. + assertThat(isRowSelectionBeneficial(8_192, 409, 25, 7_350, INT32, false, false, 4, 1)).isFalse(); + + // Boolean row-level skipping requires a selection-aware packed decoder. + assertThat(isRowSelectionBeneficial(8_192, 409, 16, 500, BOOLEAN, false, false, 1, 1)).isFalse(); + + // Nullable fixed-width readers use a 20% limit. + assertThat(isRowSelectionBeneficial(8_192, 1_638, 16, 500, INT32, false, true, 4, 1)).isTrue(); + assertThat(isRowSelectionBeneficial(8_192, 1_639, 16, 500, INT32, false, true, 4, 1)).isFalse(); + assertThat(isRowSelectionBeneficial(8_192, 1_638, 16, 500, FIXED_LEN_BYTE_ARRAY, false, true, 16, 1)).isTrue(); + + // Dictionary indexes share a 10% limit across physical value types. + assertThat(isRowSelectionBeneficial(8_192, 819, 16, 500, INT32, true, false, 1, 1)).isTrue(); + assertThat(isRowSelectionBeneficial(8_192, 820, 16, 500, INT32, true, false, 1, 1)).isFalse(); + assertThat(isRowSelectionBeneficial(8_192, 819, 16, 500, BINARY, true, false, 1, 1)).isTrue(); + assertThat(isRowSelectionBeneficial(8_192, 819, 16, 500, BINARY, true, true, 1, 1)).isTrue(); + + // Plain binary values use a 25% limit independent of value width. + assertThat(isRowSelectionBeneficial(8_192, 2_048, 16, 400, BINARY, false, false, 256, 50)).isTrue(); + assertThat(isRowSelectionBeneficial(8_192, 2_049, 16, 400, BINARY, false, false, 256, 50)).isFalse(); + assertThat(isRowSelectionBeneficial(8_192, 2_048, 16, 400, BINARY, false, true, 256, 50)).isTrue(); + + // Very wide values use row-level decoding only for small projections. + assertThat(isRowSelectionBeneficial(8_192, 2_048, 16, 400, BINARY, false, false, 1_024, 4)).isTrue(); + assertThat(isRowSelectionBeneficial(8_192, 2_048, 16, 400, BINARY, false, false, 1_024, 5)).isFalse(); + } + + @Test + public void testSelectedPositionsPushdownDisabled() + throws IOException + { + int rowCount = 4_096; + List columnNames = ImmutableList.of("column"); + List types = ImmutableList.of(BIGINT); + ParquetReaderOptions readerOptions = ParquetReaderOptions.builder() + .withSelectedPositionsPushdownEnabled(false) + .build(); + ParquetDataSource dataSource = new TestingParquetDataSource( + writeParquetFile( + ParquetWriterOptions.builder().build(), + types, + columnNames, + generateInputPages(types, rowCount, 1)), + readerOptions); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + try (ParquetReader reader = createParquetReader( + dataSource, + parquetMetadata, + readerOptions, + newSimpleAggregatedMemoryContext(), + types, + columnNames, + TupleDomain.all(), + true)) { + SourcePage page = reader.nextPage(); + while (page.getPositionCount() < 1_024) { + page = reader.nextPage(); + } + + int originalPositionCount = page.getPositionCount(); + int[] positions = IntStream.range(400, 440).toArray(); + assertThat(page.trySelectPositions(positions, 0, positions.length)).isFalse(); + assertThat(page.getPositionCount()).isEqualTo(originalPositionCount); + assertThat(reader.getSelectedPositionsPushdownCount()).isZero(); + assertThat(reader.getMetrics().getMetrics()).containsEntry(SELECTED_POSITIONS_PUSHDOWNS, new LongCount(0)); + } + } + @Test public void testColumnReaderMemoryUsage() throws IOException @@ -243,13 +335,19 @@ public void testSelectPositionsOnLoadedBlocks() page.selectPositions(new int[] {1, 3, 5, 7}, 0, 4); assertThat(blockValues(page.getBlock(0))).containsExactly(firstRow + 1, firstRow + 3, firstRow + 5, firstRow + 7); + assertThat(page.getSizeInBytes()).isEqualTo(page.getBlock(0).getSizeInBytes()); // select again with positions relative to the previous selection page.selectPositions(new int[] {1, 2}, 0, 2); assertThat(page.getPositionCount()).isEqualTo(2); // columnA was loaded before the second selection, columnB is loaded after it assertThat(blockValues(page.getBlock(0))).containsExactly(firstRow + 3, firstRow + 5); + assertThat(page.getSizeInBytes()).isEqualTo(page.getBlock(0).getSizeInBytes()); assertThat(blockValues(page.getBlock(1))).containsExactly((firstRow + 3) * 10, (firstRow + 5) * 10); + + SourcePage nextPage = reader.nextPage(); + assertThat(blockValues(nextPage.getBlock(1))) + .startsWith((firstRow + 8) * 10, (firstRow + 9) * 10, (firstRow + 10) * 10); } } @@ -307,6 +405,447 @@ private static ParquetReader createParquetReaderWithRowNumbers( Optional.empty()); } + @Test + public void testRepeatedSelectionsWithInterleavedColumnLoads() + throws IOException + { + int rowCount = 65_536; + List columnNames = ImmutableList.of("filter_a", "filter_b", "projected"); + List types = ImmutableList.of(BIGINT, BIGINT, BIGINT); + BlockBuilder filterA = BIGINT.createFixedSizeBlockBuilder(rowCount); + BlockBuilder filterB = BIGINT.createFixedSizeBlockBuilder(rowCount); + BlockBuilder projected = BIGINT.createFixedSizeBlockBuilder(rowCount); + for (int position = 0; position < rowCount; position++) { + BIGINT.writeLong(filterA, position); + BIGINT.writeLong(filterB, position * 10L); + BIGINT.writeLong(projected, position * 100L); + } + + ParquetDataSource dataSource = new TestingParquetDataSource( + writeParquetFile( + ParquetWriterOptions.builder() + .setMaxPageValueCount(1_024) + .build(), + types, + columnNames, + ImmutableList.of(new Page(rowCount, filterA.build(), filterB.build(), projected.build()))), + ParquetReaderOptions.defaultOptions()); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + try (ParquetReader reader = createParquetReader( + dataSource, + parquetMetadata, + ParquetReaderOptions.defaultOptions(), + newSimpleAggregatedMemoryContext(), + types, + columnNames, + TupleDomain.all(), + true)) { + for (SourcePage page = reader.nextPage(); page != null; page = reader.nextPage()) { + Block firstFilterBlock = page.getBlock(0); + int[] firstSelection = IntStream.range(0, page.getPositionCount()) + .filter(position -> BIGINT.getLong(firstFilterBlock, position) % 512 < 8) + .toArray(); + if (firstSelection.length == 0) { + continue; + } + page.selectPositions(firstSelection, 0, firstSelection.length); + + Block secondFilterBlock = page.getBlock(1); + int[] secondSelection = IntStream.range(0, page.getPositionCount()) + .filter(position -> (BIGINT.getLong(secondFilterBlock, position) / 10) % 2 == 0) + .toArray(); + if (secondSelection.length == 0) { + continue; + } + assertThat(page.trySelectPositions(secondSelection, 0, secondSelection.length)).isTrue(); + + Block projectedBlock = page.getBlock(2); + for (int position = 0; position < page.getPositionCount(); position++) { + assertThat(BIGINT.getLong(projectedBlock, position)) + .isEqualTo(BIGINT.getLong(page.getBlock(0), position) * 100); + } + } + } + } + + @Test + public void testUnselectedReadsShareDictionary() + throws IOException + { + int rowCount = 8_192; + int dictionarySize = 1_024; + List types = ImmutableList.of(VARCHAR); + List columnNames = ImmutableList.of("value"); + BlockBuilder values = VARCHAR.createBlockBuilder(null, rowCount); + for (int position = 0; position < rowCount; position++) { + VARCHAR.writeSlice(values, utf8Slice("%04d".formatted(position % dictionarySize) + "x".repeat(60))); + } + ParquetDataSource dataSource = new TestingParquetDataSource( + writeParquetFile( + ParquetWriterOptions.builder().setMaxPageValueCount(rowCount).build(), + types, + columnNames, + ImmutableList.of(new Page(values.build()))), + ParquetReaderOptions.defaultOptions()); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + try (ParquetReader reader = createParquetReader(dataSource, parquetMetadata, newSimpleAggregatedMemoryContext(), types, columnNames)) { + int firstPosition = 0; + SourcePage page = reader.nextPage(); + while (page.getPositionCount() < 128) { + firstPosition += page.getPositionCount(); + page = reader.nextPage(); + } + DictionaryBlock first = (DictionaryBlock) page.getBlock(0); + DictionaryBlock second = (DictionaryBlock) reader.nextPage().getBlock(0); + assertThat(first.getDictionary().getPositionCount()).isEqualTo(dictionarySize); + assertThat(second.getDictionary()).isSameAs(first.getDictionary()); + assertThat(VARCHAR.getSlice(first, 0).toStringUtf8()).isEqualTo("%04d".formatted(firstPosition % dictionarySize) + "x".repeat(60)); + assertThat(VARCHAR.getSlice(second, 0).toStringUtf8()).isEqualTo("%04d".formatted((firstPosition + first.getPositionCount()) % dictionarySize) + "x".repeat(60)); + } + } + + @Test + public void testAcceptedSelectionCompactsLoadedSelectedAndFallbackBlocks() + throws IOException + { + int rowCount = 4_096; + ArrayType arrayType = new ArrayType(BIGINT); + List columnNames = ImmutableList.of("payload", "filter", "fallback", "nested"); + List types = ImmutableList.of(VARCHAR, BIGINT, BooleanType.BOOLEAN, arrayType); + BlockBuilder payload = VARCHAR.createBlockBuilder(null, rowCount); + BlockBuilder filter = BIGINT.createFixedSizeBlockBuilder(rowCount); + BlockBuilder fallback = BooleanType.BOOLEAN.createFixedSizeBlockBuilder(rowCount); + for (int position = 0; position < rowCount; position++) { + VARCHAR.writeSlice(payload, utf8Slice("%08d".formatted(position) + "x".repeat(248))); + BIGINT.writeLong(filter, position); + BooleanType.BOOLEAN.writeBoolean(fallback, position % 2 == 0); + } + Block nested = createArrayBlock(Optional.empty(), rowCount); + + ParquetDataSource dataSource = new TestingParquetDataSource( + writeParquetFile( + ParquetWriterOptions.builder() + .setMaxPageValueCount(rowCount) + .build(), + types, + columnNames, + ImmutableList.of(new Page(rowCount, payload.build(), filter.build(), fallback.build(), nested))), + ParquetReaderOptions.defaultOptions()); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + try (ParquetReader reader = createParquetReader(dataSource, parquetMetadata, newSimpleAggregatedMemoryContext(), types, columnNames)) { + long firstRow = 0; + SourcePage page = reader.nextPage(); + while (page.getPositionCount() < 1_024) { + firstRow += page.getPositionCount(); + page = reader.nextPage(); + } + + page.getBlock(1); + int[] positions = IntStream.range(400, 500).toArray(); + assertThat(page.trySelectPositions(positions, 0, positions.length)).isTrue(); + + Block selectedPayload = page.getBlock(0); + Block selectedFilter = page.getBlock(1); + Block selectedFallback = page.getBlock(2); + Block selectedNested = page.getBlock(3); + for (int index = 0; index < positions.length; index++) { + long expectedValue = firstRow + positions[index]; + assertThat(VARCHAR.getSlice(selectedPayload, index).toStringUtf8()) + .isEqualTo("%08d".formatted(expectedValue) + "x".repeat(248)); + assertThat(BIGINT.getLong(selectedFilter, index)).isEqualTo(expectedValue); + assertThat(BooleanType.BOOLEAN.getBoolean(selectedFallback, index)).isEqualTo(expectedValue % 2 == 0); + assertThat(arrayType.getObjectValue(selectedNested, index)) + .isEqualTo(arrayType.getObjectValue(nested, toIntExact(expectedValue))); + } + for (Block block : ImmutableList.of(selectedPayload, selectedFilter, selectedFallback, selectedNested)) { + assertThat(block.getPositionCount()).isEqualTo(positions.length); + assertThat(block.getRetainedSizeInBytes()).isLessThanOrEqualTo(block.getSizeInBytes() * 2); + } + } + } + + @Test + public void testSelectNonAscendingPositions() + throws IOException + { + List columnNames = ImmutableList.of("column"); + List types = ImmutableList.of(BIGINT); + int rowCount = 100; + BlockBuilder values = BIGINT.createFixedSizeBlockBuilder(rowCount); + for (int i = 0; i < rowCount; i++) { + BIGINT.writeLong(values, i); + } + + ParquetDataSource dataSource = new TestingParquetDataSource( + writeParquetFile( + ParquetWriterOptions.builder().build(), + types, + columnNames, + ImmutableList.of(new Page(values.build()))), + ParquetReaderOptions.defaultOptions()); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + try (ParquetReader reader = createParquetReader(dataSource, parquetMetadata, newSimpleAggregatedMemoryContext(), types, columnNames)) { + long firstRow = 0; + SourcePage page = reader.nextPage(); + while (page.getPositionCount() < 4) { + firstRow += page.getPositionCount(); + page = reader.nextPage(); + } + + int[] positions = new int[13]; + positions[10] = 3; + positions[11] = 1; + positions[12] = 3; + page.selectPositions(positions, 10, 3); + assertThat(page.getPositionCount()).isEqualTo(3); + assertThat(blockValues(page.getBlock(0))).containsExactly(firstRow + 3, firstRow + 1, firstRow + 3); + assertThat(reader.getSelectedPositionsPushdownCount()).isZero(); + assertThat(reader.getSelectedPositionsFallbackCount()).isOne(); + } + } + + @Test + public void testSingleSelectedRunAtPageEdgeFallsBack() + throws IOException + { + int rowCount = 4_096; + List columnNames = ImmutableList.of("column"); + List types = ImmutableList.of(BIGINT); + BlockBuilder values = BIGINT.createFixedSizeBlockBuilder(rowCount); + for (int position = 0; position < rowCount; position++) { + BIGINT.writeLong(values, position); + } + + ParquetDataSource dataSource = new TestingParquetDataSource( + writeParquetFile( + ParquetWriterOptions.builder() + .setMaxPageValueCount(rowCount) + .build(), + types, + columnNames, + ImmutableList.of(new Page(values.build()))), + ParquetReaderOptions.defaultOptions()); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + try (ParquetReader reader = createParquetReader(dataSource, parquetMetadata, newSimpleAggregatedMemoryContext(), types, columnNames)) { + long firstRow = 0; + SourcePage page = reader.nextPage(); + while (page.getPositionCount() < 1_024) { + firstRow += page.getPositionCount(); + page = reader.nextPage(); + } + + int[] edgeRun = IntStream.range(0, 40).toArray(); + page.selectPositions(edgeRun, 0, edgeRun.length); + long edgeFirstRow = firstRow; + assertThat(blockValues(page.getBlock(0))).containsExactlyElementsOf( + IntStream.range(0, 40).mapToLong(position -> edgeFirstRow + position).boxed().toList()); + assertThat(reader.getSelectedPositionsPushdownCount()).isZero(); + assertThat(reader.getSelectedPositionsFallbackCount()).isOne(); + assertThat(reader.getMetrics().getMetrics()).containsEntry(SELECTED_POSITIONS_PUSHDOWNS, new LongCount(0)); + + firstRow += 1_024; + page = reader.nextPage(); + int[] interiorRun = IntStream.range(400, 480).toArray(); + page.selectPositions(interiorRun, 0, interiorRun.length); + long interiorFirstRow = firstRow; + assertThat(blockValues(page.getBlock(0))).containsExactlyElementsOf( + IntStream.range(400, 480).mapToLong(position -> interiorFirstRow + position).boxed().toList()); + assertThat(reader.getSelectedPositionsPushdownCount()).isOne(); + assertThat(reader.getSelectedPositionsFallbackCount()).isOne(); + assertThat(reader.getMetrics().getMetrics()).containsEntry(SELECTED_POSITIONS_PUSHDOWNS, new LongCount(1)); + } + } + + @Test + public void testSelectPositionOutsidePage() + throws IOException + { + List columnNames = ImmutableList.of("column"); + List types = ImmutableList.of(BIGINT); + ParquetDataSource dataSource = new TestingParquetDataSource( + writeParquetFile( + ParquetWriterOptions.builder().build(), + types, + columnNames, + generateInputPages(types, 100, 1)), + ParquetReaderOptions.defaultOptions()); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + try (ParquetReader reader = createParquetReader(dataSource, parquetMetadata, newSimpleAggregatedMemoryContext(), types, columnNames)) { + SourcePage page = reader.nextPage(); + assertThatThrownBy(() -> page.selectPositions(new int[] {page.getPositionCount()}, 0, 1)) + .isInstanceOf(IndexOutOfBoundsException.class); + } + } + + @Test + public void testSelectedPositionsAreCopied() + throws IOException + { + List columnNames = ImmutableList.of("column"); + List types = ImmutableList.of(BIGINT); + ParquetDataSource dataSource = new TestingParquetDataSource( + writeParquetFile( + ParquetWriterOptions.builder().build(), + types, + columnNames, + generateInputPages(types, 100, 1)), + ParquetReaderOptions.defaultOptions()); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + try (ParquetReader reader = createParquetReader( + dataSource, + parquetMetadata, + ParquetReaderOptions.defaultOptions(), + newSimpleAggregatedMemoryContext(), + types, + columnNames, + TupleDomain.all(), + true)) { + long firstRow = 0; + SourcePage page = reader.nextPage(); + while (page.getPositionCount() < 4) { + firstRow += page.getPositionCount(); + page = reader.nextPage(); + } + + // No unloaded column can benefit, so the source must remain unchanged. + int[] offeredPositions = {1, 3}; + int originalPositionCount = page.getPositionCount(); + page.getBlock(0); + assertThat(page.trySelectPositions(offeredPositions, 0, offeredPositions.length)).isFalse(); + assertThat(page.getPositionCount()).isEqualTo(originalPositionCount); + + // Use an unloaded page for the existing ownership check below. + firstRow += originalPositionCount; + page = reader.nextPage(); + while (page.getPositionCount() < 4) { + firstRow += page.getPositionCount(); + page = reader.nextPage(); + } + + int mandatoryPagePositionCount = page.getPositionCount(); + int[] positions = {1, 3}; + page.selectPositions(positions, 0, positions.length); + Arrays.fill(positions, 0); + + assertThat(blockValues(page.getBlock(0))).containsExactly(firstRow + 1, firstRow + 3); + + // An accepted optional selection must also retain its own positions. + firstRow += mandatoryPagePositionCount; + page = reader.nextPage(); + while (page.getPositionCount() < 4) { + firstRow += page.getPositionCount(); + page = reader.nextPage(); + } + + int[] offeredSelection = {-1, 1, 3, -1}; + assertThat(page.trySelectPositions(offeredSelection, 1, 2)).isTrue(); + Arrays.fill(offeredSelection, 0); + + long[] retainedBytes = {0}; + page.retainedBytesForEachPart((_, bytes) -> retainedBytes[0] += bytes); + assertThat(retainedBytes[0]).isEqualTo(page.getRetainedSizeInBytes()); + + assertThat(blockValues(page.getBlock(0))).containsExactly(firstRow + 1, firstRow + 3); + } + } + + @Test + public void testSelectedPositionsUpdateAdaptiveBatchSizeBasedOnInputRows() + throws IOException + { + List columnNames = ImmutableList.of("payload"); + List types = ImmutableList.of(VARCHAR); + int rowCount = 3_000; + BlockBuilder values = VARCHAR.createBlockBuilder(null, rowCount); + for (int position = 0; position < rowCount; position++) { + VARCHAR.writeSlice(values, utf8Slice("%08d".formatted(position) + "x".repeat(248))); + } + + ParquetReaderOptions readerOptions = ParquetReaderOptions.builder() + .withMaxReadBlockSize(DataSize.ofBytes(200_000)) + .withMaxReadBlockRowCount(rowCount) + .build(); + ParquetDataSource dataSource = new TestingParquetDataSource( + writeParquetFile( + ParquetWriterOptions.builder() + .setMaxPageValueCount(rowCount) + .build(), + types, + columnNames, + ImmutableList.of(new Page(values.build()))), + readerOptions); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + try (ParquetReader reader = createParquetReader(dataSource, parquetMetadata, readerOptions, newSimpleAggregatedMemoryContext(), types, columnNames, TupleDomain.all())) { + for (int expectedBatchSize : new int[] {1, 2, 4, 8, 16}) { + SourcePage page = reader.nextPage(); + assertThat(page.getPositionCount()).isEqualTo(expectedBatchSize); + assertThat(page.getBlock(0).getPositionCount()).isEqualTo(expectedBatchSize); + } + for (int expectedBatchSize : new int[] {32, 64, 128}) { + SourcePage page = reader.nextPage(); + assertThat(page.getPositionCount()).isEqualTo(expectedBatchSize); + page.selectPositions(new int[] {0}, 0, 1); + assertThat(page.getBlock(0).getPositionCount()).isEqualTo(1); + } + + assertThat(reader.nextPage().getPositionCount()).isEqualTo(256); + } + } + + @Test + public void testSelectPositionsWithColumnIndexRowRanges() + throws Exception + { + File parquetFile = new File(Resources.getResource("lineitem_sorted_by_shipdate/data.parquet").toURI()); + List columnNames = ImmutableList.of("l_shipdate", "l_commitdate"); + List types = ImmutableList.of(DATE, DATE); + TupleDomain predicate = TupleDomain.withColumnDomains( + ImmutableMap.of( + "l_shipdate", Domain.multipleValues(DATE, ImmutableList.of(LocalDate.of(1993, 1, 1).toEpochDay(), LocalDate.of(1997, 1, 1).toEpochDay())), + "l_commitdate", Domain.create(ValueSet.ofRanges(Range.greaterThan(DATE, LocalDate.of(1995, 1, 1).toEpochDay())), false))); + + try (ParquetReader expectedReader = createFilteredParquetReader(parquetFile, types, columnNames, predicate); + ParquetReader selectedReader = createFilteredParquetReader(parquetFile, types, columnNames, predicate)) { + SourcePage expectedPage = expectedReader.nextPage(); + SourcePage selectedPage = selectedReader.nextPage(); + int selectedRowCount = 0; + while (expectedPage != null) { + assertThat(selectedPage).isNotNull(); + assertThat(selectedPage.getPositionCount()).isEqualTo(expectedPage.getPositionCount()); + + int[] positions = IntStream.range(0, expectedPage.getPositionCount()) + .filter(position -> position % 5 <= 1) + .toArray(); + Block expectedShipDate = expectedPage.getBlock(0).copyPositions(positions, 0, positions.length); + Block expectedCommitDate = expectedPage.getBlock(1).copyPositions(positions, 0, positions.length); + selectedPage.selectPositions(positions, 0, positions.length); + assertThat(dateValues(selectedPage.getBlock(0))).isEqualTo(dateValues(expectedShipDate)); + assertThat(dateValues(selectedPage.getBlock(1))).isEqualTo(dateValues(expectedCommitDate)); + selectedRowCount += positions.length; + + expectedPage = expectedReader.nextPage(); + selectedPage = selectedReader.nextPage(); + } + assertThat(selectedPage).isNull(); + assertThat(selectedRowCount).isPositive(); + } + } + + private static ParquetReader createFilteredParquetReader(File parquetFile, List types, List columnNames, TupleDomain predicate) + throws IOException + { + ParquetDataSource dataSource = new FileParquetDataSource(parquetFile, ParquetReaderOptions.defaultOptions()); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); + return createParquetReader( + dataSource, + parquetMetadata, + ParquetReaderOptions.defaultOptions(), + newSimpleAggregatedMemoryContext(), + types, + columnNames, + predicate); + } + private static List blockValues(Block block) { ImmutableList.Builder values = ImmutableList.builder(); @@ -316,6 +855,15 @@ private static List blockValues(Block block) return values.build(); } + private static List dateValues(Block block) + { + ImmutableList.Builder values = ImmutableList.builder(); + for (int position = 0; position < block.getPositionCount(); position++) { + values.add(DATE.getLong(block, position)); + } + return values.build(); + } + @Test public void testBackwardsCompatibleRepeatedStringField() throws Exception diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetReaderConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetReaderConfig.java index a64ad781b9be..4deb946e1914 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetReaderConfig.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetReaderConfig.java @@ -160,6 +160,21 @@ public DataSize getSmallFileThreshold() return options.getSmallFileThreshold(); } + public boolean isSelectedPositionsPushdownEnabled() + { + return options.isSelectedPositionsPushdownEnabled(); + } + + @Config("parquet.selected-positions-pushdown-enabled") + @ConfigDescription("Enable pushing selected positions into Parquet column readers") + public ParquetReaderConfig setSelectedPositionsPushdownEnabled(boolean selectedPositionsPushdownEnabled) + { + options = ParquetReaderOptions.builder(options) + .withSelectedPositionsPushdownEnabled(selectedPositionsPushdownEnabled) + .build(); + return this; + } + @Config("parquet.experimental.vectorized-decoding.enabled") @ConfigDescription("Enable using Java Vector API for faster decoding of parquet files") public ParquetReaderConfig setVectorizedDecodingEnabled(boolean vectorizedDecodingEnabled) diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestParquetPageSkipping.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestParquetPageSkipping.java index cd9c2ad27125..a44afd215918 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestParquetPageSkipping.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestParquetPageSkipping.java @@ -20,12 +20,15 @@ import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.spi.security.ConnectorIdentity; +import io.trino.testing.MaterializedResult; +import io.trino.testing.MaterializedRow; import io.trino.testing.QueryRunner; import org.junit.jupiter.api.Test; import java.io.IOException; import java.io.OutputStream; import java.net.URL; +import java.util.List; import java.util.UUID; import static io.trino.plugin.hive.TestingHiveUtils.getConnectorService; @@ -101,6 +104,46 @@ public void testPageSkipping() testPageSkipping("custkey", "smallint", new Object[][] {{4, 634, 640, 1493}}); } + @Test + public void testSelectedPositionsPushdown() + { + String tableName = "test_selected_positions_pushdown_" + randomNameSuffix(); + try { + assertUpdate( + """ + CREATE TABLE %s ( + filter_key bigint, + payload varchar, + bucket integer) + WITH ( + format = 'PARQUET', + bucketed_by = ARRAY['bucket'], + bucket_count = 1, + sorted_by = ARRAY['filter_key']) + """.formatted(tableName)); + assertUpdate( + """ + INSERT INTO %s + SELECT value, CAST(value AS varchar) || 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', 0 + FROM UNNEST(sequence(0, 8191)) AS t(value) + """.formatted(tableName), + 8_192); + + MaterializedResult allRows = computeActual("SELECT filter_key, payload FROM " + tableName + " ORDER BY filter_key"); + List expectedRows = allRows.getMaterializedRows().stream() + .filter(row -> ((long) row.getField(0) % 1_024) >= 400) + .filter(row -> ((long) row.getField(0) % 1_024) <= 407) + .toList(); + MaterializedResult selectedRows = computeActual( + "SELECT filter_key, payload FROM " + tableName + " WHERE filter_key % 1024 BETWEEN 400 AND 407 ORDER BY filter_key"); + + assertThat(selectedRows.getMaterializedRows()).containsExactlyElementsOf(expectedRows); + } + finally { + assertUpdate("DROP TABLE IF EXISTS " + tableName); + } + } + private void testPageSkipping(String sortByColumn, String sortByColumnType, Object[][] valuesArray) { String tableName = "test_page_skipping_" + randomNameSuffix(); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/parquet/TestParquetReaderConfig.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/parquet/TestParquetReaderConfig.java index 6552cac474b8..7d9c3c2472f7 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/parquet/TestParquetReaderConfig.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/parquet/TestParquetReaderConfig.java @@ -39,6 +39,7 @@ public void testDefaults() .setUseColumnIndex(true) .setUseBloomFilter(true) .setSmallFileThreshold(DataSize.of(3, MEGABYTE)) + .setSelectedPositionsPushdownEnabled(true) .setVectorizedDecodingEnabled(true) .setFooterReadSize(DataSize.of(48, KILOBYTE)) .setMaxFooterReadSize(DataSize.of(15, MEGABYTE)) @@ -57,6 +58,7 @@ public void testExplicitPropertyMappings() .put("parquet.use-column-index", "false") .put("parquet.use-bloom-filter", "false") .put("parquet.small-file-threshold", "1kB") + .put("parquet.selected-positions-pushdown-enabled", "false") .put("parquet.experimental.vectorized-decoding.enabled", "false") .put("parquet.footer-read-size", "57kB") .put("parquet.max-footer-read-size", "25MB") @@ -72,6 +74,7 @@ public void testExplicitPropertyMappings() .setUseColumnIndex(false) .setUseBloomFilter(false) .setSmallFileThreshold(DataSize.of(1, KILOBYTE)) + .setSelectedPositionsPushdownEnabled(false) .setVectorizedDecodingEnabled(false) .setFooterReadSize(DataSize.of(57, KILOBYTE)) .setMaxFooterReadSize(DataSize.of(25, MEGABYTE)) diff --git a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java index 97e2ad3d3c7a..e91681257165 100644 --- a/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java +++ b/plugin/trino-redshift/src/test/java/io/trino/plugin/redshift/TestRedshiftUnload.java @@ -98,7 +98,7 @@ void testUnload() .filter(summary -> summary.getOperatorType().startsWith("TableScanOperator")) .map(operatorStat -> operatorStat.getConnectorMetrics().getMetrics()) .flatMap(metrics -> metrics.keySet().stream()) - .filter(key -> !key.startsWith("ParquetReader")) + .filter(key -> key.startsWith("s3://")) .collect(toImmutableList()); unloadedPaths.forEach(path -> assertThat(path).matches("%s/.*/.*/.*.parquet.*".formatted(S3_UNLOAD_ROOT))); String unloadedFilePath = unloadedPaths.getFirst(); From 722626ef1f7b9fc76b2f0d655de036a85d29ec50 Mon Sep 17 00:00:00 2001 From: Dain Sundstrom Date: Sun, 12 Jul 2026 20:38:42 -0700 Subject: [PATCH 16/18] Benchmark Parquet position selection The selection policy spans several cost regimes and cannot be justified by a single type or selectivity measurement. Provide a Zstandard benchmark covering encoding, value width, null layout, selection locality, page geometry, and projected column count. Compare full reads, forced selection, and the production decision while exposing page and policy counters. --- .../BenchmarkParquetSelectedPositions.java | 1097 +++++++++++++++++ ...TestBenchmarkParquetSelectedPositions.java | 343 ++++++ 2 files changed, 1440 insertions(+) create mode 100644 lib/trino-parquet/src/test/java/io/trino/parquet/BenchmarkParquetSelectedPositions.java create mode 100644 lib/trino-parquet/src/test/java/io/trino/parquet/TestBenchmarkParquetSelectedPositions.java diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/BenchmarkParquetSelectedPositions.java b/lib/trino-parquet/src/test/java/io/trino/parquet/BenchmarkParquetSelectedPositions.java new file mode 100644 index 000000000000..91df58b489e4 --- /dev/null +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/BenchmarkParquetSelectedPositions.java @@ -0,0 +1,1097 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.sun.management.ThreadMXBean; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import io.airlift.units.DataSize; +import io.trino.memory.context.AggregatedMemoryContext; +import io.trino.parquet.metadata.ParquetMetadata; +import io.trino.parquet.reader.ChunkedInputStream; +import io.trino.parquet.reader.MetadataReader; +import io.trino.parquet.reader.PageReader; +import io.trino.parquet.reader.ParquetReader; +import io.trino.parquet.reader.TestingParquetDataSource; +import io.trino.parquet.writer.ParquetWriter; +import io.trino.parquet.writer.ParquetWriterOptions; +import io.trino.spi.Page; +import io.trino.spi.PageBuilder; +import io.trino.spi.block.Block; +import io.trino.spi.block.BlockBuilder; +import io.trino.spi.connector.SourcePage; +import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.type.BigintType; +import io.trino.spi.type.BooleanType; +import io.trino.spi.type.DateType; +import io.trino.spi.type.DecimalType; +import io.trino.spi.type.DoubleType; +import io.trino.spi.type.Int128; +import io.trino.spi.type.IntegerType; +import io.trino.spi.type.RealType; +import io.trino.spi.type.Type; +import io.trino.spi.type.VarcharType; +import org.apache.parquet.format.CompressionCodec; +import org.openjdk.jmh.annotations.AuxCounters; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.runner.RunnerException; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import static com.google.common.base.Preconditions.checkState; +import static io.airlift.units.DataSize.Unit.MEGABYTE; +import static io.trino.jmh.Benchmarks.benchmark; +import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static io.trino.parquet.ParquetTestUtils.createParquetReader; +import static io.trino.parquet.ParquetTestUtils.createParquetWriter; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static java.lang.Math.round; +import static java.lang.Math.toIntExact; + +@State(Scope.Thread) +@OutputTimeUnit(TimeUnit.SECONDS) +@Fork(1) +@Warmup(iterations = 8, time = 1) +@Measurement(iterations = 8, time = 1) +public class BenchmarkParquetSelectedPositions +{ + private static final int SELECTION_WINDOW_SIZE = 8 * 1024; + private static final ThreadMXBean THREAD_MX_BEAN = (ThreadMXBean) ManagementFactory.getThreadMXBean(); + + private long comparisonInvocation; + private Slice parquetFile; + private ParquetMetadata parquetMetadata; + private ParquetReaderOptions readerOptions; + private List columnDataTypes; + private List columnTypes; + private int[] columnNullPercentages; + private List columnNames; + private int[] selectedFilePositions; + private int[][] dataPageEndPositions; + private long selectedDataPageCount; + private long totalDataPageCount; + private long inputSize; + private long parquetUncompressedSize; + private long dictionaryPageBytes; + private int dictionaryEncodedColumnChunks; + private int nonDictionaryEncodedColumnChunks; + + @Param("ZSTD") + public CompressionCodec compression; + + @Param({"BOOLEAN", "INTEGER", "BIGINT", "DOUBLE", "VARCHAR", "DATE", "REAL", "DECIMAL_9", "DECIMAL_18", "DECIMAL_30"}) + public DataType dataType; + + @Param("RANDOM") + public ValueShape valueShape; + + @Param("ADAPTIVE") + public PushdownDecision pushdownDecision; + + @Param("1") + public int columnCount; + + @Param("0") + public int preloadedColumnCount; + + @Param("HOMOGENEOUS") + public ColumnLayout columnLayout; + + @Param("RUNS") + public SelectionShape selectionShape; + + @Param({"0.05", "0.10", "0.25"}) + public double selectivity; + + @Param({"1", "20", "24", "128"}) + public int runCount; + + @Param("65536") + public int rowCount; + + @Param("4096") + public int pageValueCount; + + @Param("256") + public int payloadWidth; + + @Param("32") + public int entropyBytes; + + @Param("0") + public int dictionaryCardinality; + + @Param("1") + public int dictionaryRunLength; + + @Param("0") + public int nullPercentage; + + @Param("RANDOM") + public NullShape nullShape; + + @Param("16") + public int nullRunCount; + + @Setup + public void setup() + throws IOException + { + if (selectivity < 0 || selectivity > 1) { + throw new IllegalArgumentException("selectivity must be between 0 and 1"); + } + if (payloadWidth < 1 || entropyBytes < 0 || entropyBytes > payloadWidth) { + throw new IllegalArgumentException("entropyBytes must be between 0 and payloadWidth"); + } + if (dictionaryCardinality < 0) { + throw new IllegalArgumentException("dictionaryCardinality is negative"); + } + if (dictionaryRunLength < 1) { + throw new IllegalArgumentException("dictionaryRunLength must be positive"); + } + if (nullPercentage < 0 || nullPercentage > 100) { + throw new IllegalArgumentException("nullPercentage must be between 0 and 100"); + } + if (nullRunCount < 1) { + throw new IllegalArgumentException("nullRunCount must be positive"); + } + if (columnCount < 1) { + throw new IllegalArgumentException("columnCount must be positive"); + } + if (preloadedColumnCount < 0 || preloadedColumnCount > columnCount) { + throw new IllegalArgumentException("preloadedColumnCount must be between zero and columnCount"); + } + if (pageValueCount < 1) { + throw new IllegalArgumentException("pageValueCount must be positive"); + } + ImmutableList.Builder columnDataTypesBuilder = ImmutableList.builderWithExpectedSize(columnCount); + ImmutableList.Builder columnTypesBuilder = ImmutableList.builderWithExpectedSize(columnCount); + columnNullPercentages = new int[columnCount]; + ImmutableList.Builder columnNamesBuilder = ImmutableList.builderWithExpectedSize(columnCount); + for (int channel = 0; channel < columnCount; channel++) { + DataType columnDataType = columnLayout.dataType(dataType, channel); + columnDataTypesBuilder.add(columnDataType); + columnTypesBuilder.add(columnDataType.getType()); + columnNullPercentages[channel] = columnLayout.nullPercentage(nullPercentage, channel); + columnNamesBuilder.add("payload_" + channel); + } + columnDataTypes = columnDataTypesBuilder.build(); + columnTypes = columnTypesBuilder.build(); + columnNames = columnNamesBuilder.build(); + + List pages = createInputPages(); + inputSize = pages.stream().mapToLong(Page::getSizeInBytes).sum(); + ParquetWriterOptions writerOptions = ParquetWriterOptions.builder() + .setMaxBlockSize(DataSize.of(64, MEGABYTE)) + .setMaxPageSize(DataSize.of(16, MEGABYTE)) + .setMaxPageValueCount(pageValueCount) + .build(); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + try (ParquetWriter writer = createParquetWriter(output, writerOptions, columnTypes, columnNames, compression)) { + for (Page page : pages) { + writer.write(page); + } + } + parquetFile = Slices.wrappedBuffer(output.toByteArray()); + readerOptions = ParquetReaderOptions.builder() + .withMaxBufferSize(DataSize.of(64, MEGABYTE)) + .build(); + try (TestingParquetDataSource dataSource = new TestingParquetDataSource(parquetFile, readerOptions)) { + parquetMetadata = MetadataReader.readFooter(dataSource, readerOptions, Optional.empty(), Optional.empty()); + } + dataPageEndPositions = readDataPageEndPositions(); + selectedFilePositions = createSelectedFilePositions(); + selectedDataPageCount = countSelectedPages(dataPageEndPositions, selectedFilePositions); + totalDataPageCount = Arrays.stream(dataPageEndPositions) + .mapToLong(pageEndPositions -> pageEndPositions.length) + .sum(); + parquetUncompressedSize = parquetMetadata.getBlocks().stream() + .flatMap(block -> block.columns().stream()) + .mapToLong(column -> column.getTotalUncompressedSize()) + .sum(); + dictionaryEncodedColumnChunks = toIntExact(parquetMetadata.getBlocks().stream() + .flatMap(block -> block.columns().stream()) + .filter(column -> column.getEncodingStats().hasDictionaryEncodedPages()) + .count()); + dictionaryPageBytes = parquetMetadata.getBlocks().stream() + .flatMap(block -> block.columns().stream()) + .filter(column -> column.getDictionaryPageOffset() > 0) + .mapToLong(column -> column.getFirstDataPageOffset() - column.getDictionaryPageOffset()) + .sum(); + nonDictionaryEncodedColumnChunks = toIntExact(parquetMetadata.getBlocks().stream() + .flatMap(block -> block.columns().stream()) + .filter(column -> column.getEncodingStats().hasNonDictionaryEncodedPages()) + .count()); + } + + @AuxCounters(AuxCounters.Type.EVENTS) + @State(Scope.Thread) + public static class DataCounters + { + public long unselectedDataPages; + public long dictionaryEncodedColumnChunks; + public long dictionaryPageBytes; + public long fullDecodeDataPagesRead; + public long fullDecodeAllocatedBytes; + public long fullDecodeOutputBytes; + public long fullDecodeFirstOutputRetainedBytes; + public long fullDecodeOutputRetainedBytes; + public long fullDecodeNanos; + public long fullDecodeSourcePages; + public long inputBytes; + public long nonDictionaryEncodedColumnChunks; + public long parquetFileBytes; + public long parquetUncompressedBytes; + public long rejectedRows; + public long readOperations; + public long selectedDataPages; + public long selectedPushdownDataPagesRead; + public long selectedPushdownAllocatedBytes; + public long selectedPushdownOutputBytes; + public long selectedPushdownFirstOutputRetainedBytes; + public long selectedPushdownOutputRetainedBytes; + public long selectedPushdownNanos; + public long selectedPushdownSourcePages; + public long selectedPositionsFallbacks; + public long selectedPositionsPushdowns; + public long selectedRuns; + public long selectedRows; + public long totalDataPages; + } + + @Benchmark + public long fullDecodeThenSelect(DataCounters counters) + throws IOException + { + return read(false, counters); + } + + @Benchmark + public long selectedPositionsPushdown(DataCounters counters) + throws IOException + { + return read(true, counters); + } + + @Benchmark + public long pairedComparison(DataCounters counters) + throws IOException + { + // Keep both paths in the same invocation and alternate their order to minimize workstation drift. + long fullDecodeChecksum; + long selectedPushdownChecksum; + if ((comparisonInvocation++ & 1) == 0) { + long start = System.nanoTime(); + long allocatedBytes = THREAD_MX_BEAN.getCurrentThreadAllocatedBytes(); + fullDecodeChecksum = read(false, counters); + counters.fullDecodeAllocatedBytes += THREAD_MX_BEAN.getCurrentThreadAllocatedBytes() - allocatedBytes; + counters.fullDecodeNanos += System.nanoTime() - start; + + start = System.nanoTime(); + allocatedBytes = THREAD_MX_BEAN.getCurrentThreadAllocatedBytes(); + selectedPushdownChecksum = read(true, counters); + counters.selectedPushdownAllocatedBytes += THREAD_MX_BEAN.getCurrentThreadAllocatedBytes() - allocatedBytes; + counters.selectedPushdownNanos += System.nanoTime() - start; + } + else { + long start = System.nanoTime(); + long allocatedBytes = THREAD_MX_BEAN.getCurrentThreadAllocatedBytes(); + selectedPushdownChecksum = read(true, counters); + counters.selectedPushdownAllocatedBytes += THREAD_MX_BEAN.getCurrentThreadAllocatedBytes() - allocatedBytes; + counters.selectedPushdownNanos += System.nanoTime() - start; + + start = System.nanoTime(); + allocatedBytes = THREAD_MX_BEAN.getCurrentThreadAllocatedBytes(); + fullDecodeChecksum = read(false, counters); + counters.fullDecodeAllocatedBytes += THREAD_MX_BEAN.getCurrentThreadAllocatedBytes() - allocatedBytes; + counters.fullDecodeNanos += System.nanoTime() - start; + } + checkState(fullDecodeChecksum == selectedPushdownChecksum, "Checksums do not match"); + return fullDecodeChecksum; + } + + private long read(boolean pushdown, DataCounters counters) + throws IOException + { + AggregatedMemoryContext memoryContext = newSimpleAggregatedMemoryContext(); + TestingParquetDataSource dataSource = new TestingParquetDataSource(parquetFile, readerOptions); + long[] checksums = new long[columnCount]; + int globalOffset = 0; + long selectedRuns = 0; + long selectedRows = 0; + long sourcePageCount = 0; + int dataPagesRead; + int selectedPositionsFallbacks; + int selectedPositionsPushdowns; + try (ParquetReader reader = createParquetReader( + dataSource, + parquetMetadata, + readerOptions, + memoryContext, + columnTypes, + columnNames, + TupleDomain.all(), + pushdownDecision == PushdownDecision.FORCE)) { + for (SourcePage page = reader.nextPage(); page != null; page = reader.nextPage()) { + sourcePageCount++; + int originalPositionCount = page.getPositionCount(); + int[] positions = selectedPositions(globalOffset, originalPositionCount); + + for (int channel = columnCount - preloadedColumnCount; channel < columnCount; channel++) { + page.getBlock(channel); + } + + if (positions.length == 0) { + globalOffset += originalPositionCount; + continue; + } + + boolean allPositionsSelected = positions.length == originalPositionCount; + boolean selectionApplied = pushdown + && !allPositionsSelected + && page.trySelectPositions(positions, 0, positions.length); + + for (int channel = 0; channel < columnCount; channel++) { + Block block = page.getBlock(channel); + if (!selectionApplied && !allPositionsSelected) { + block = block.copyPositions(positions, 0, positions.length); + } + if (pushdown) { + counters.selectedPushdownOutputBytes += block.getSizeInBytes(); + counters.selectedPushdownOutputRetainedBytes += block.getRetainedSizeInBytes(); + if (channel == 0) { + counters.selectedPushdownFirstOutputRetainedBytes += block.getRetainedSizeInBytes(); + } + } + else { + counters.fullDecodeOutputBytes += block.getSizeInBytes(); + counters.fullDecodeOutputRetainedBytes += block.getRetainedSizeInBytes(); + if (channel == 0) { + counters.fullDecodeFirstOutputRetainedBytes += block.getRetainedSizeInBytes(); + } + } + for (int position = 0; position < block.getPositionCount(); position++) { + checksums[channel] = columnDataTypes.get(channel).checksum(checksums[channel], block, position); + } + } + selectedRuns += countRuns(positions); + selectedRows += positions.length; + globalOffset += originalPositionCount; + } + dataPagesRead = reader.getDataPageReadCount(); + selectedPositionsFallbacks = reader.getSelectedPositionsFallbackCount(); + selectedPositionsPushdowns = reader.getSelectedPositionsPushdownCount(); + } + memoryContext.close(); + + counters.dictionaryEncodedColumnChunks += dictionaryEncodedColumnChunks; + counters.dictionaryPageBytes += dictionaryPageBytes; + counters.inputBytes += inputSize; + counters.nonDictionaryEncodedColumnChunks += nonDictionaryEncodedColumnChunks; + counters.parquetFileBytes += parquetFile.length(); + counters.parquetUncompressedBytes += parquetUncompressedSize; + counters.unselectedDataPages += totalDataPageCount - selectedDataPageCount; + counters.rejectedRows += rowCount - selectedRows; + counters.readOperations++; + counters.selectedDataPages += selectedDataPageCount; + if (pushdown) { + counters.selectedPositionsFallbacks += selectedPositionsFallbacks; + counters.selectedPositionsPushdowns += selectedPositionsPushdowns; + counters.selectedPushdownDataPagesRead += dataPagesRead; + counters.selectedPushdownSourcePages += sourcePageCount; + } + else { + counters.fullDecodeDataPagesRead += dataPagesRead; + counters.fullDecodeSourcePages += sourcePageCount; + } + counters.selectedRuns += selectedRuns; + counters.selectedRows += selectedRows; + counters.totalDataPages += totalDataPageCount; + long checksum = 0; + for (long channelChecksum : checksums) { + checksum = 31 * checksum + channelChecksum; + } + return checksum; + } + + private int[][] readDataPageEndPositions() + throws IOException + { + List> pageEndPositions = new ArrayList<>(); + int[] rowOffsets = new int[columnCount]; + for (int channel = 0; channel < columnCount; channel++) { + pageEndPositions.add(new ArrayList<>()); + } + AggregatedMemoryContext memoryContext = newSimpleAggregatedMemoryContext(); + try (TestingParquetDataSource dataSource = new TestingParquetDataSource(parquetFile, readerOptions)) { + for (var block : parquetMetadata.getBlocks()) { + for (int channel = 0; channel < columnCount; channel++) { + var columnMetadata = block.columns().get(channel); + DiskRange diskRange = new DiskRange(columnMetadata.getStartingPos(), columnMetadata.getTotalSize()); + ChunkedInputStream input = dataSource.planRead(ImmutableListMultimap.of(channel, diskRange), memoryContext).get(channel); + try (input) { + PageReader pageReader = PageReader.createPageReader( + dataSource.getId(), + input, + columnMetadata, + parquetMetadata.getFileMetaData().getSchema().getColumns().get(channel), + null, + Optional.ofNullable(parquetMetadata.getFileMetaData().getCreatedBy()), + Optional.empty(), + readerOptions.getMaxPageReadSize().toBytes()); + pageReader.readDictionaryPage(); + while (pageReader.hasNext()) { + DataPage page = pageReader.getNextPage(); + rowOffsets[channel] += page.getValueCount(); + pageEndPositions.get(channel).add(rowOffsets[channel]); + pageReader.skipNextPage(); + } + } + } + } + } + memoryContext.close(); + for (int channel = 0; channel < columnCount; channel++) { + checkState(rowOffsets[channel] == rowCount, "Column %s data pages contain %s rows, expected %s", channel, rowOffsets[channel], rowCount); + } + return pageEndPositions.stream() + .map(positions -> positions.stream().mapToInt(Integer::intValue).toArray()) + .toArray(int[][]::new); + } + + private int[] createSelectedFilePositions() + { + if (selectionShape == SelectionShape.PAGE_ALIGNED || selectionShape == SelectionShape.PAGE_SHIFTED) { + return createPageSelectedFilePositions(selectionShape == SelectionShape.PAGE_SHIFTED); + } + int[] positions = new int[rowCount]; + int selectedCount = 0; + if (selectionShape == SelectionShape.RUNS) { + for (int windowOffset = 0; windowOffset < rowCount; windowOffset += SELECTION_WINDOW_SIZE) { + int windowSize = min(SELECTION_WINDOW_SIZE, rowCount - windowOffset); + int[] windowPositions = selectionShape.select(windowOffset, windowSize, selectivity, runCount, pageValueCount); + for (int position : windowPositions) { + positions[selectedCount++] = windowOffset + position; + } + } + } + else { + int[] filePositions = selectionShape.select(0, rowCount, selectivity, runCount, pageValueCount); + System.arraycopy(filePositions, 0, positions, 0, filePositions.length); + selectedCount = filePositions.length; + } + return Arrays.copyOf(positions, selectedCount); + } + + private int[] createPageSelectedFilePositions(boolean shifted) + { + boolean[] selected = new boolean[rowCount]; + int pageStart = 0; + int[] firstColumnPageEndPositions = dataPageEndPositions[0]; + for (int pageIndex = 0; pageIndex < firstColumnPageEndPositions.length; pageIndex++) { + int pageEnd = firstColumnPageEndPositions[pageIndex]; + if (SelectionShape.isSelectedPage(pageIndex, selectivity)) { + int shift = shifted ? (pageEnd - pageStart) / 2 : 0; + for (int position = pageStart; position < pageEnd; position++) { + selected[Math.floorMod(position - shift, rowCount)] = true; + } + } + pageStart = pageEnd; + } + + int[] positions = new int[rowCount]; + int selectedCount = 0; + for (int position = 0; position < rowCount; position++) { + if (selected[position]) { + positions[selectedCount++] = position; + } + } + return Arrays.copyOf(positions, selectedCount); + } + + private int[] selectedPositions(int pageOffset, int positionCount) + { + int start = lowerBound(selectedFilePositions, pageOffset); + int end = lowerBound(selectedFilePositions, pageOffset + positionCount); + int[] positions = new int[end - start]; + for (int index = start; index < end; index++) { + positions[index - start] = selectedFilePositions[index] - pageOffset; + } + return positions; + } + + private static int lowerBound(int[] values, int value) + { + int low = 0; + int high = values.length; + while (low < high) { + int middle = (low + high) >>> 1; + if (values[middle] < value) { + low = middle + 1; + } + else { + high = middle; + } + } + return low; + } + + private static long countSelectedPages(int[][] dataPageEndPositions, int[] selectedPositions) + { + long count = 0; + for (int[] columnPageEndPositions : dataPageEndPositions) { + boolean[] selectedPages = new boolean[columnPageEndPositions.length]; + for (int selectedPosition : selectedPositions) { + selectedPages[lowerBound(columnPageEndPositions, selectedPosition + 1)] = true; + } + for (boolean selected : selectedPages) { + if (selected) { + count++; + } + } + } + return count; + } + + private static int countRuns(int[] positions) + { + int runs = positions.length == 0 ? 0 : 1; + for (int index = 1; index < positions.length; index++) { + if (positions[index] != positions[index - 1] + 1) { + runs++; + } + } + return runs; + } + + private List createInputPages() + { + List pages = new ArrayList<>(); + PageBuilder pageBuilder = PageBuilder.withMaxPageSize(toIntExact(DataSize.of(64, MEGABYTE).toBytes()), columnTypes); + Slice[][] dictionaries = new Slice[columnCount][dictionaryCardinality]; + for (int row = 0; row < rowCount; row++) { + pageBuilder.declarePosition(); + for (int channel = 0; channel < columnCount; channel++) { + BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(channel); + int valueId = row * columnCount + channel; + if (nullShape.isNull(row, rowCount, columnNullPercentages[channel], nullRunCount, valueId)) { + blockBuilder.appendNull(); + } + else { + if (dictionaryCardinality > 0) { + valueId = ((row / dictionaryRunLength) * columnCount + channel) % dictionaryCardinality; + } + columnDataTypes.get(channel).write(this, blockBuilder, valueId, dictionaries[channel]); + } + } + + if ((row + 1) % pageValueCount == 0 || pageBuilder.isFull()) { + pages.add(pageBuilder.build()); + pageBuilder.reset(); + } + } + if (!pageBuilder.isEmpty()) { + pages.add(pageBuilder.build()); + } + return pages; + } + + public enum DataType + { + BOOLEAN(BooleanType.BOOLEAN) { + @Override + void write(BenchmarkParquetSelectedPositions benchmark, BlockBuilder builder, int valueId, Slice[] dictionary) + { + BooleanType.BOOLEAN.writeBoolean(builder, (mix(valueId) & 1) == 0); + } + + @Override + long checksum(long checksum, Block block, int position) + { + if (block.isNull(position)) { + return 31 * checksum + 1; + } + return 31 * checksum + (BooleanType.BOOLEAN.getBoolean(block, position) ? 1231 : 1237); + } + }, + INTEGER(IntegerType.INTEGER) { + @Override + void write(BenchmarkParquetSelectedPositions benchmark, BlockBuilder builder, int valueId, Slice[] dictionary) + { + IntegerType.INTEGER.writeInt(builder, (int) benchmark.valueShape.value(valueId)); + } + + @Override + long checksum(long checksum, Block block, int position) + { + return block.isNull(position) ? 31 * checksum + 1 : 31 * checksum + IntegerType.INTEGER.getInt(block, position); + } + }, + BIGINT(BigintType.BIGINT) { + @Override + void write(BenchmarkParquetSelectedPositions benchmark, BlockBuilder builder, int valueId, Slice[] dictionary) + { + BigintType.BIGINT.writeLong(builder, benchmark.valueShape.value(valueId)); + } + + @Override + long checksum(long checksum, Block block, int position) + { + return block.isNull(position) ? 31 * checksum + 1 : 31 * checksum + BigintType.BIGINT.getLong(block, position); + } + }, + DOUBLE(DoubleType.DOUBLE) { + @Override + void write(BenchmarkParquetSelectedPositions benchmark, BlockBuilder builder, int valueId, Slice[] dictionary) + { + DoubleType.DOUBLE.writeDouble(builder, benchmark.valueShape.doubleValue(valueId)); + } + + @Override + long checksum(long checksum, Block block, int position) + { + return block.isNull(position) ? 31 * checksum + 1 : 31 * checksum + Double.doubleToLongBits(DoubleType.DOUBLE.getDouble(block, position)); + } + }, + VARCHAR(VarcharType.VARCHAR) { + @Override + void write(BenchmarkParquetSelectedPositions benchmark, BlockBuilder builder, int valueId, Slice[] dictionary) + { + Slice value = dictionary.length == 0 ? null : dictionary[valueId]; + if (value == null) { + value = benchmark.createPayload(valueId); + if (dictionary.length > 0) { + dictionary[valueId] = value; + } + } + VarcharType.VARCHAR.writeSlice(builder, value); + } + + @Override + long checksum(long checksum, Block block, int position) + { + if (block.isNull(position)) { + return 31 * checksum + 1; + } + Slice value = VarcharType.VARCHAR.getSlice(block, position); + checksum = 31 * checksum + value.length(); + checksum = 31 * checksum + value.getByte(0); + return 31 * checksum + value.getByte(value.length() - 1); + } + }, + DATE(DateType.DATE) { + @Override + void write(BenchmarkParquetSelectedPositions benchmark, BlockBuilder builder, int valueId, Slice[] dictionary) + { + DateType.DATE.writeLong(builder, Math.floorMod(valueId, 365_000)); + } + + @Override + long checksum(long checksum, Block block, int position) + { + return block.isNull(position) ? 31 * checksum + 1 : 31 * checksum + DateType.DATE.getLong(block, position); + } + }, + REAL(RealType.REAL) { + @Override + void write(BenchmarkParquetSelectedPositions benchmark, BlockBuilder builder, int valueId, Slice[] dictionary) + { + float value = (float) benchmark.valueShape.doubleValue(valueId); + RealType.REAL.writeLong(builder, Float.floatToRawIntBits(value)); + } + + @Override + long checksum(long checksum, Block block, int position) + { + return block.isNull(position) ? 31 * checksum + 1 : 31 * checksum + Float.floatToRawIntBits(RealType.REAL.getFloat(block, position)); + } + }, + DECIMAL_9(DecimalType.createDecimalType(9, 2)) { + @Override + void write(BenchmarkParquetSelectedPositions benchmark, BlockBuilder builder, int valueId, Slice[] dictionary) + { + ((DecimalType) getType()).writeLong(builder, Math.floorMod(benchmark.valueShape.value(valueId), 100_000_000L)); + } + + @Override + long checksum(long checksum, Block block, int position) + { + return block.isNull(position) ? 31 * checksum + 1 : 31 * checksum + ((DecimalType) getType()).getLong(block, position); + } + }, + DECIMAL_18(DecimalType.createDecimalType(18, 2)) { + @Override + void write(BenchmarkParquetSelectedPositions benchmark, BlockBuilder builder, int valueId, Slice[] dictionary) + { + ((DecimalType) getType()).writeLong(builder, Math.floorMod(benchmark.valueShape.value(valueId), 100_000_000_000_000_000L)); + } + + @Override + long checksum(long checksum, Block block, int position) + { + return block.isNull(position) ? 31 * checksum + 1 : 31 * checksum + ((DecimalType) getType()).getLong(block, position); + } + }, + DECIMAL_30(DecimalType.createDecimalType(30, 2)) { + @Override + void write(BenchmarkParquetSelectedPositions benchmark, BlockBuilder builder, int valueId, Slice[] dictionary) + { + ((DecimalType) getType()).writeObject(builder, Int128.valueOf(benchmark.valueShape.value(valueId))); + } + + @Override + long checksum(long checksum, Block block, int position) + { + if (block.isNull(position)) { + return 31 * checksum + 1; + } + Int128 value = (Int128) ((DecimalType) getType()).getObject(block, position); + checksum = 31 * checksum + value.getHigh(); + return 31 * checksum + value.getLow(); + } + }; + + private final Type type; + + DataType(Type type) + { + this.type = type; + } + + Type getType() + { + return type; + } + + abstract void write(BenchmarkParquetSelectedPositions benchmark, BlockBuilder builder, int valueId, Slice[] dictionary); + + abstract long checksum(long checksum, Block block, int position); + } + + public enum ValueShape + { + RANDOM { + @Override + long value(int valueId) + { + return mix(valueId); + } + + @Override + double doubleValue(int valueId) + { + return (mix(valueId) >>> 11) * 0x1.0p-53; + } + }, + SEQUENTIAL { + @Override + long value(int valueId) + { + return valueId; + } + }, + LOW_CARDINALITY { + @Override + long value(int valueId) + { + return valueId % 16; + } + }, + SORTED_RUNS { + @Override + long value(int valueId) + { + return valueId / 32; + } + }; + + abstract long value(int valueId); + + double doubleValue(int valueId) + { + return value(valueId); + } + } + + public enum PushdownDecision + { + ADAPTIVE, + FORCE, + } + + public enum ColumnLayout + { + HOMOGENEOUS { + @Override + DataType dataType(DataType firstColumnType, int channel) + { + return firstColumnType; + } + + @Override + int nullPercentage(int configuredNullPercentage, int channel) + { + return configuredNullPercentage; + } + }, + FIRST_TYPE_THEN_BIGINT { + @Override + DataType dataType(DataType firstColumnType, int channel) + { + return channel == 0 ? firstColumnType : DataType.BIGINT; + } + + @Override + int nullPercentage(int configuredNullPercentage, int channel) + { + return channel == 0 ? 0 : configuredNullPercentage; + } + }; + + abstract DataType dataType(DataType firstColumnType, int channel); + + abstract int nullPercentage(int configuredNullPercentage, int channel); + } + + private Slice createPayload(int valueId) + { + byte[] bytes = new byte[payloadWidth]; + Arrays.fill(bytes, (byte) 'x'); + long state = mix(valueId); + for (int index = 0; index < entropyBytes; index++) { + if ((index & 7) == 0) { + state = mix(state + index); + } + bytes[index] = (byte) (state >>> ((index & 7) * Byte.SIZE)); + } + return Slices.wrappedBuffer(bytes); + } + + private static long mix(long value) + { + value = (value ^ (value >>> 30)) * 0xBF58476D1CE4E5B9L; + value = (value ^ (value >>> 27)) * 0x94D049BB133111EBL; + return value ^ (value >>> 31); + } + + private static int unsignedRemainder(long value, int divisor) + { + return (int) Long.remainderUnsigned(value, divisor); + } + + public enum SelectionShape + { + RUNS { + @Override + int[] select(int globalOffset, int positionCount, double selectivity, int requestedRunCount, int pageValueCount) + { + int selectedCount = (int) round(positionCount * selectivity); + if (selectivity > 0 && positionCount > 0) { + selectedCount = max(1, selectedCount); + } + selectedCount = min(positionCount, selectedCount); + if (selectedCount == 0) { + return new int[0]; + } + + int runs = min(max(1, requestedRunCount), min(selectedCount, positionCount - selectedCount + 1)); + int[] positions = new int[selectedCount]; + int[] gaps = new int[runs + 1]; + int freeGaps = positionCount - selectedCount - (runs - 1); + Arrays.fill(gaps, freeGaps / gaps.length); + for (int gap = 0; gap < freeGaps % gaps.length; gap++) { + gaps[gap]++; + } + + int outputIndex = 0; + int position = gaps[0]; + for (int run = 0; run < runs; run++) { + int runLength = selectedCount / runs + (run < selectedCount % runs ? 1 : 0); + for (int index = 0; index < runLength; index++) { + positions[outputIndex++] = position++; + } + if (run + 1 < runs) { + position += 1 + gaps[run + 1]; + } + } + return positions; + } + }, + SKEWED_RUNS { + @Override + int[] select(int globalOffset, int positionCount, double selectivity, int requestedRunCount, int pageValueCount) + { + int selectedCount = selectedCount(positionCount, selectivity); + if (selectedCount == 0) { + return new int[0]; + } + + int runs = min(max(1, requestedRunCount), min(selectedCount, positionCount - selectedCount + 1)); + int[] positions = new int[selectedCount]; + int position = positionCount - selectedCount - (runs - 1); + int longRunLength = selectedCount - (runs - 1); + int outputIndex = 0; + for (int index = 0; index < longRunLength; index++) { + positions[outputIndex++] = position++; + } + for (int run = 1; run < runs; run++) { + position++; + positions[outputIndex++] = position++; + } + return positions; + } + }, + CONCENTRATED { + @Override + int[] select(int globalOffset, int positionCount, double selectivity, int requestedRunCount, int pageValueCount) + { + int selectedCount = selectedCount(positionCount, selectivity); + int firstPosition = (positionCount - selectedCount) / 2; + int[] positions = new int[selectedCount]; + for (int index = 0; index < selectedCount; index++) { + positions[index] = firstPosition + index; + } + return positions; + } + }, + DISTRIBUTED { + @Override + int[] select(int globalOffset, int positionCount, double selectivity, int requestedRunCount, int pageValueCount) + { + int selectedCount = selectedCount(positionCount, selectivity); + if (selectedCount == 0) { + return new int[0]; + } + int[] positions = new int[selectedCount]; + for (int index = 0; index < selectedCount; index++) { + positions[index] = toIntExact(((long) index * positionCount + positionCount / 2) / selectedCount); + } + return positions; + } + }, + PAGE_ALIGNED { + @Override + int[] select(int globalOffset, int positionCount, double selectivity, int requestedRunCount, int pageValueCount) + { + if (selectivity == 0) { + return new int[0]; + } + int[] positions = new int[positionCount]; + int selectedCount = 0; + for (int position = 0; position < positionCount; position++) { + long globalPosition = (long) globalOffset + position; + if (isSelectedPage(globalPosition / pageValueCount, selectivity)) { + positions[selectedCount++] = position; + } + } + return Arrays.copyOf(positions, selectedCount); + } + }, + PAGE_SHIFTED { + @Override + int[] select(int globalOffset, int positionCount, double selectivity, int requestedRunCount, int pageValueCount) + { + if (selectivity == 0) { + return new int[0]; + } + int shift = max(1, pageValueCount / 2); + int[] positions = new int[positionCount]; + int selectedCount = 0; + for (int position = 0; position < positionCount; position++) { + long globalPosition = (long) globalOffset + position + shift; + if (isSelectedPage(globalPosition / pageValueCount, selectivity)) { + positions[selectedCount++] = position; + } + } + return Arrays.copyOf(positions, selectedCount); + } + }; + + abstract int[] select(int globalOffset, int positionCount, double selectivity, int requestedRunCount, int pageValueCount); + + private static int selectedCount(int positionCount, double selectivity) + { + int selectedCount = (int) round(positionCount * selectivity); + if (selectivity > 0 && positionCount > 0) { + selectedCount = max(1, selectedCount); + } + return min(positionCount, selectedCount); + } + + private static boolean isSelectedPage(long pageIndex, double selectivity) + { + return (long) Math.floor((pageIndex + 1) * selectivity + 0.5) > (long) Math.floor(pageIndex * selectivity + 0.5); + } + } + + public enum NullShape + { + RANDOM { + @Override + boolean isNull(int row, int rowCount, int nullPercentage, int nullRunCount, int valueId) + { + return unsignedRemainder(mix(valueId), 100) < nullPercentage; + } + }, + CLUSTERED { + @Override + boolean isNull(int row, int rowCount, int nullPercentage, int nullRunCount, int valueId) + { + int nullCount = rowCount * nullPercentage / 100; + int firstNull = (rowCount - nullCount) / 2; + return row >= firstNull && row < firstNull + nullCount; + } + }, + RUNS { + @Override + boolean isNull(int row, int rowCount, int nullPercentage, int nullRunCount, int valueId) + { + int cycleLength = max(1, rowCount / nullRunCount); + return row % cycleLength < cycleLength * nullPercentage / 100; + } + }; + + abstract boolean isNull(int row, int rowCount, int nullPercentage, int nullRunCount, int valueId); + } + + static void main() + throws RunnerException + { + benchmark(BenchmarkParquetSelectedPositions.class) + .withOptions(optionsBuilder -> optionsBuilder + .addProfiler(GCProfiler.class) + .jvmArgsAppend("-Xmx4g", "-Xms4g", "--add-modules=jdk.incubator.vector")) + .run(); + } +} diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/TestBenchmarkParquetSelectedPositions.java b/lib/trino-parquet/src/test/java/io/trino/parquet/TestBenchmarkParquetSelectedPositions.java new file mode 100644 index 000000000000..8f9b0b535f7f --- /dev/null +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/TestBenchmarkParquetSelectedPositions.java @@ -0,0 +1,343 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet; + +import org.apache.parquet.format.CompressionCodec; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestBenchmarkParquetSelectedPositions +{ + @Test + public void testBenchmark() + throws Exception + { + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.BOOLEAN, BenchmarkParquetSelectedPositions.SelectionShape.RUNS, 8, 0, 10, 1); + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.INTEGER, BenchmarkParquetSelectedPositions.SelectionShape.PAGE_ALIGNED, 1, 0, 0, 1); + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.BIGINT, BenchmarkParquetSelectedPositions.SelectionShape.RUNS, 8, 0, 10, 3); + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.DOUBLE, BenchmarkParquetSelectedPositions.SelectionShape.RUNS, 8, 0, 10, 1); + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.VARCHAR, BenchmarkParquetSelectedPositions.SelectionShape.RUNS, 8, 0, 10, 1); + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.DATE, BenchmarkParquetSelectedPositions.SelectionShape.RUNS, 8, 0, 10, 1); + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.REAL, BenchmarkParquetSelectedPositions.SelectionShape.RUNS, 8, 0, 10, 1); + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.DECIMAL_9, BenchmarkParquetSelectedPositions.SelectionShape.RUNS, 8, 0, 10, 1); + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.DECIMAL_18, BenchmarkParquetSelectedPositions.SelectionShape.RUNS, 8, 0, 10, 1); + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.DECIMAL_30, BenchmarkParquetSelectedPositions.SelectionShape.RUNS, 8, 0, 10, 1); + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.VARCHAR, BenchmarkParquetSelectedPositions.SelectionShape.RUNS, 8, 32, 10, 1); + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.VARCHAR, BenchmarkParquetSelectedPositions.SelectionShape.CONCENTRATED, 8, 0, 10, 1); + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.VARCHAR, BenchmarkParquetSelectedPositions.SelectionShape.DISTRIBUTED, 8, 0, 10, 1); + assertBenchmark(BenchmarkParquetSelectedPositions.DataType.VARCHAR, BenchmarkParquetSelectedPositions.SelectionShape.PAGE_SHIFTED, 8, 0, 10, 1); + assertBenchmark( + BenchmarkParquetSelectedPositions.DataType.VARCHAR, + BenchmarkParquetSelectedPositions.SelectionShape.RUNS, + 1, + 0, + 50, + 3, + BenchmarkParquetSelectedPositions.ColumnLayout.FIRST_TYPE_THEN_BIGINT); + } + + @Test + public void testPageSelectionShapesPreserveSelectivity() + { + assertThat(BenchmarkParquetSelectedPositions.SelectionShape.PAGE_ALIGNED.select(0, 1024, 0.75, 1, 128)).hasSize(768); + assertThat(BenchmarkParquetSelectedPositions.SelectionShape.PAGE_SHIFTED.select(0, 1024, 0.75, 1, 128)).hasSize(768); + assertThat(BenchmarkParquetSelectedPositions.SelectionShape.SKEWED_RUNS.select(0, 1024, 0.10, 20, 128)) + .hasSize(102) + .isSorted(); + } + + @Test + public void testAdaptivePushdownPolicy() + throws Exception + { + assertAdaptiveDecision(BenchmarkParquetSelectedPositions.DataType.BOOLEAN, 0.05, 0, 0, false); + assertAdaptiveDecision(BenchmarkParquetSelectedPositions.DataType.INTEGER, 0.05, 0, 0, true); + assertAdaptiveDecision(BenchmarkParquetSelectedPositions.DataType.INTEGER, 0.10, 0, 0, false); + assertAdaptiveDecision(BenchmarkParquetSelectedPositions.DataType.INTEGER, 0.05, 20, 0, 0, true); + assertAdaptiveDecision(BenchmarkParquetSelectedPositions.DataType.INTEGER, 0.05, 24, 0, 0, true); + assertAdaptiveDecision(BenchmarkParquetSelectedPositions.DataType.INTEGER, 0.05, 28, 0, 0, false); + assertAdaptiveDecision(BenchmarkParquetSelectedPositions.DataType.INTEGER, 0.20, 0, 50, true); + assertAdaptiveDecision(BenchmarkParquetSelectedPositions.DataType.VARCHAR, 0.25, 0, 0, true); + assertAdaptiveDecision(BenchmarkParquetSelectedPositions.DataType.VARCHAR, 0.50, 0, 0, false); + assertAdaptiveDecision(BenchmarkParquetSelectedPositions.DataType.VARCHAR, 0.10, 16, 0, true); + assertAdaptiveDecision(BenchmarkParquetSelectedPositions.DataType.DECIMAL_30, 0.20, 0, 50, true); + assertAdaptiveDecision(BenchmarkParquetSelectedPositions.DataType.DECIMAL_30, 0.25, 0, 50, false); + } + + @Test + public void testAdaptivePagePushdownPolicy() + throws Exception + { + assertAdaptivePageDecision(BenchmarkParquetSelectedPositions.DataType.INTEGER, 0, false); + assertAdaptivePageDecision(BenchmarkParquetSelectedPositions.DataType.BIGINT, 0, false); + assertAdaptivePageDecision(BenchmarkParquetSelectedPositions.DataType.DATE, 0, false); + assertAdaptivePageDecision(BenchmarkParquetSelectedPositions.DataType.REAL, 0, false); + assertAdaptivePageDecision(BenchmarkParquetSelectedPositions.DataType.DOUBLE, 0, true); + assertAdaptivePageDecision(BenchmarkParquetSelectedPositions.DataType.DECIMAL_18, 0, true); + assertAdaptivePageDecision(BenchmarkParquetSelectedPositions.DataType.DOUBLE, 1_024, false); + } + + @Test + public void testLoadedColumnsDoNotDisableWideBinaryPushdown() + throws Exception + { + BenchmarkParquetSelectedPositions benchmark = new BenchmarkParquetSelectedPositions(); + benchmark.compression = CompressionCodec.ZSTD; + benchmark.dataType = BenchmarkParquetSelectedPositions.DataType.VARCHAR; + benchmark.valueShape = BenchmarkParquetSelectedPositions.ValueShape.RANDOM; + benchmark.pushdownDecision = BenchmarkParquetSelectedPositions.PushdownDecision.ADAPTIVE; + benchmark.columnCount = 5; + benchmark.preloadedColumnCount = 4; + benchmark.columnLayout = BenchmarkParquetSelectedPositions.ColumnLayout.FIRST_TYPE_THEN_BIGINT; + benchmark.selectionShape = BenchmarkParquetSelectedPositions.SelectionShape.RUNS; + benchmark.selectivity = 0.25; + benchmark.runCount = 16; + benchmark.rowCount = 8_192; + benchmark.pageValueCount = 8_192; + benchmark.payloadWidth = 1_024; + benchmark.entropyBytes = 16; + benchmark.dictionaryCardinality = 0; + benchmark.dictionaryRunLength = 1; + benchmark.nullPercentage = 0; + benchmark.nullShape = BenchmarkParquetSelectedPositions.NullShape.RANDOM; + benchmark.nullRunCount = 16; + benchmark.setup(); + + BenchmarkParquetSelectedPositions.DataCounters counters = new BenchmarkParquetSelectedPositions.DataCounters(); + benchmark.selectedPositionsPushdown(counters); + assertThat(counters.selectedPositionsPushdowns).isPositive(); + } + + @Test + public void testSkewedRunsFallBack() + throws Exception + { + assertAdaptiveDecision( + BenchmarkParquetSelectedPositions.DataType.INTEGER, + BenchmarkParquetSelectedPositions.SelectionShape.RUNS, + true); + assertAdaptiveDecision( + BenchmarkParquetSelectedPositions.DataType.INTEGER, + BenchmarkParquetSelectedPositions.SelectionShape.SKEWED_RUNS, + false); + } + + private static void assertAdaptivePageDecision( + BenchmarkParquetSelectedPositions.DataType dataType, + int dictionaryCardinality, + boolean expectedPushdown) + throws Exception + { + BenchmarkParquetSelectedPositions benchmark = new BenchmarkParquetSelectedPositions(); + benchmark.compression = CompressionCodec.ZSTD; + benchmark.dataType = dataType; + benchmark.valueShape = BenchmarkParquetSelectedPositions.ValueShape.RANDOM; + benchmark.pushdownDecision = BenchmarkParquetSelectedPositions.PushdownDecision.ADAPTIVE; + benchmark.columnCount = 1; + benchmark.preloadedColumnCount = 0; + benchmark.columnLayout = BenchmarkParquetSelectedPositions.ColumnLayout.HOMOGENEOUS; + benchmark.selectionShape = BenchmarkParquetSelectedPositions.SelectionShape.PAGE_ALIGNED; + benchmark.selectivity = 0.25; + benchmark.runCount = 1; + benchmark.rowCount = 65_536; + benchmark.pageValueCount = 4_096; + benchmark.payloadWidth = 64; + benchmark.entropyBytes = 16; + benchmark.dictionaryCardinality = dictionaryCardinality; + benchmark.dictionaryRunLength = 1; + benchmark.nullPercentage = 0; + benchmark.nullShape = BenchmarkParquetSelectedPositions.NullShape.RANDOM; + benchmark.nullRunCount = 16; + benchmark.setup(); + + BenchmarkParquetSelectedPositions.DataCounters baselineCounters = new BenchmarkParquetSelectedPositions.DataCounters(); + BenchmarkParquetSelectedPositions.DataCounters pushdownCounters = new BenchmarkParquetSelectedPositions.DataCounters(); + assertThat(benchmark.selectedPositionsPushdown(pushdownCounters)) + .isEqualTo(benchmark.fullDecodeThenSelect(baselineCounters)); + if (expectedPushdown) { + assertThat(pushdownCounters.selectedPositionsPushdowns) + .as("%s dictionary %s", dataType, dictionaryCardinality) + .isPositive(); + assertThat(pushdownCounters.selectedPushdownDataPagesRead) + .as("%s dictionary %s", dataType, dictionaryCardinality) + .isLessThan(baselineCounters.fullDecodeDataPagesRead); + } + else { + assertThat(pushdownCounters.selectedPositionsPushdowns) + .as("%s dictionary %s", dataType, dictionaryCardinality) + .isZero(); + assertThat(pushdownCounters.selectedPushdownDataPagesRead) + .as("%s dictionary %s", dataType, dictionaryCardinality) + .isEqualTo(baselineCounters.fullDecodeDataPagesRead); + } + } + + private static void assertAdaptiveDecision( + BenchmarkParquetSelectedPositions.DataType dataType, + double selectivity, + int dictionaryCardinality, + int nullPercentage, + boolean expectedPushdown) + throws Exception + { + assertAdaptiveDecision(dataType, selectivity, 16, dictionaryCardinality, nullPercentage, expectedPushdown); + } + + private static void assertAdaptiveDecision( + BenchmarkParquetSelectedPositions.DataType dataType, + double selectivity, + int runCount, + int dictionaryCardinality, + int nullPercentage, + boolean expectedPushdown) + throws Exception + { + BenchmarkParquetSelectedPositions benchmark = new BenchmarkParquetSelectedPositions(); + benchmark.compression = CompressionCodec.ZSTD; + benchmark.dataType = dataType; + benchmark.valueShape = BenchmarkParquetSelectedPositions.ValueShape.RANDOM; + benchmark.pushdownDecision = BenchmarkParquetSelectedPositions.PushdownDecision.ADAPTIVE; + benchmark.columnCount = 1; + benchmark.preloadedColumnCount = 0; + benchmark.columnLayout = BenchmarkParquetSelectedPositions.ColumnLayout.HOMOGENEOUS; + benchmark.selectionShape = BenchmarkParquetSelectedPositions.SelectionShape.RUNS; + benchmark.selectivity = selectivity; + benchmark.runCount = runCount; + benchmark.rowCount = 8_192; + benchmark.pageValueCount = 8_192; + benchmark.payloadWidth = 64; + benchmark.entropyBytes = 16; + benchmark.dictionaryCardinality = dictionaryCardinality; + benchmark.dictionaryRunLength = 1; + benchmark.nullPercentage = nullPercentage; + benchmark.nullShape = BenchmarkParquetSelectedPositions.NullShape.RANDOM; + benchmark.nullRunCount = 16; + benchmark.setup(); + + BenchmarkParquetSelectedPositions.DataCounters counters = new BenchmarkParquetSelectedPositions.DataCounters(); + benchmark.selectedPositionsPushdown(counters); + if (expectedPushdown) { + assertThat(counters.selectedPositionsPushdowns) + .as("%s selectivity %s, dictionary %s, nulls %s", dataType, selectivity, dictionaryCardinality, nullPercentage) + .isPositive(); + } + else { + assertThat(counters.selectedPositionsPushdowns) + .as("%s selectivity %s, dictionary %s, nulls %s", dataType, selectivity, dictionaryCardinality, nullPercentage) + .isZero(); + } + } + + private static void assertAdaptiveDecision( + BenchmarkParquetSelectedPositions.DataType dataType, + BenchmarkParquetSelectedPositions.SelectionShape selectionShape, + boolean expectedPushdown) + throws Exception + { + BenchmarkParquetSelectedPositions benchmark = new BenchmarkParquetSelectedPositions(); + benchmark.compression = CompressionCodec.ZSTD; + benchmark.dataType = dataType; + benchmark.valueShape = BenchmarkParquetSelectedPositions.ValueShape.RANDOM; + benchmark.pushdownDecision = BenchmarkParquetSelectedPositions.PushdownDecision.ADAPTIVE; + benchmark.columnCount = 1; + benchmark.preloadedColumnCount = 0; + benchmark.columnLayout = BenchmarkParquetSelectedPositions.ColumnLayout.HOMOGENEOUS; + benchmark.selectionShape = selectionShape; + benchmark.selectivity = 0.05; + benchmark.runCount = 20; + benchmark.rowCount = 65_536; + benchmark.pageValueCount = 65_536; + benchmark.payloadWidth = 64; + benchmark.entropyBytes = 16; + benchmark.dictionaryCardinality = 0; + benchmark.dictionaryRunLength = 1; + benchmark.nullPercentage = 0; + benchmark.nullShape = BenchmarkParquetSelectedPositions.NullShape.RANDOM; + benchmark.nullRunCount = 16; + benchmark.setup(); + + BenchmarkParquetSelectedPositions.DataCounters counters = new BenchmarkParquetSelectedPositions.DataCounters(); + benchmark.selectedPositionsPushdown(counters); + if (expectedPushdown) { + assertThat(counters.selectedPositionsPushdowns).isPositive(); + } + else { + assertThat(counters.selectedPositionsPushdowns).isZero(); + } + } + + private static void assertBenchmark( + BenchmarkParquetSelectedPositions.DataType dataType, + BenchmarkParquetSelectedPositions.SelectionShape selectionShape, + int runCount, + int dictionaryCardinality, + int nullPercentage, + int columnCount) + throws Exception + { + assertBenchmark(dataType, selectionShape, runCount, dictionaryCardinality, nullPercentage, columnCount, BenchmarkParquetSelectedPositions.ColumnLayout.HOMOGENEOUS); + } + + private static void assertBenchmark( + BenchmarkParquetSelectedPositions.DataType dataType, + BenchmarkParquetSelectedPositions.SelectionShape selectionShape, + int runCount, + int dictionaryCardinality, + int nullPercentage, + int columnCount, + BenchmarkParquetSelectedPositions.ColumnLayout columnLayout) + throws Exception + { + BenchmarkParquetSelectedPositions benchmark = new BenchmarkParquetSelectedPositions(); + benchmark.compression = CompressionCodec.ZSTD; + benchmark.dataType = dataType; + benchmark.valueShape = BenchmarkParquetSelectedPositions.ValueShape.RANDOM; + benchmark.pushdownDecision = BenchmarkParquetSelectedPositions.PushdownDecision.ADAPTIVE; + benchmark.columnCount = columnCount; + benchmark.preloadedColumnCount = 0; + benchmark.columnLayout = columnLayout; + benchmark.selectionShape = selectionShape; + benchmark.selectivity = 0.1; + benchmark.runCount = runCount; + benchmark.rowCount = 1024; + benchmark.pageValueCount = 128; + benchmark.payloadWidth = 64; + benchmark.entropyBytes = 16; + benchmark.dictionaryCardinality = dictionaryCardinality; + benchmark.dictionaryRunLength = 1; + benchmark.nullPercentage = nullPercentage; + benchmark.nullShape = BenchmarkParquetSelectedPositions.NullShape.RANDOM; + benchmark.nullRunCount = 16; + benchmark.setup(); + + BenchmarkParquetSelectedPositions.DataCounters baselineCounters = new BenchmarkParquetSelectedPositions.DataCounters(); + BenchmarkParquetSelectedPositions.DataCounters pushdownCounters = new BenchmarkParquetSelectedPositions.DataCounters(); + assertThat(benchmark.fullDecodeThenSelect(baselineCounters)) + .isEqualTo(benchmark.selectedPositionsPushdown(pushdownCounters)); + assertThat(pushdownCounters.selectedRows).isEqualTo(baselineCounters.selectedRows); + assertThat(pushdownCounters.selectedRuns).isEqualTo(baselineCounters.selectedRuns); + assertThat(pushdownCounters.selectedDataPages).isEqualTo(baselineCounters.selectedDataPages); + assertThat(pushdownCounters.dictionaryEncodedColumnChunks + pushdownCounters.nonDictionaryEncodedColumnChunks).isPositive(); + assertThat(pushdownCounters.parquetFileBytes).isPositive(); + assertThat(pushdownCounters.inputBytes).isPositive(); + assertThat(pushdownCounters.parquetUncompressedBytes).isPositive(); + assertThat(pushdownCounters.totalDataPages).isGreaterThanOrEqualTo(pushdownCounters.selectedDataPages); + assertThat(baselineCounters.fullDecodeDataPagesRead).isPositive(); + assertThat(pushdownCounters.selectedPushdownDataPagesRead).isPositive(); + if (columnLayout != BenchmarkParquetSelectedPositions.ColumnLayout.HOMOGENEOUS) { + assertThat(pushdownCounters.selectedPositionsPushdowns).isPositive(); + } + } +} From 2be9807e0965ff787db3a0e3daa6f2a9974d0114 Mon Sep 17 00:00:00 2001 From: Piotr Findeisen Date: Fri, 4 Sep 2026 19:04:15 +0200 Subject: [PATCH 17/18] Remove stray TODO leading to semantics change Currently, `concat(a_char, a_char, a_char...)` works. It works by coercing `char` values to `varchar` (trimming trailing spaces), and so is not consistent with `concat(a_char, a_char)` which retains trailing spaces. If we were to act upon the TODO being removed, we would need to add implement `concat(a_char, a_char, a_char...)` consistently with the 2-arg version, so preserving the trailing spaces. This would be semantically breaking change though. --- .../src/main/java/io/trino/operator/scalar/StringFunctions.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/StringFunctions.java b/core/trino-main/src/main/java/io/trino/operator/scalar/StringFunctions.java index b33c0cedb69c..e9c67e1062c7 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/StringFunctions.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/StringFunctions.java @@ -964,7 +964,6 @@ public static Slice toUtf8(@LiteralParameter("x") long x, @SqlType("char(x)") Sl return Chars.padSpaces(slice, toIntExact(x)); } - // TODO: implement N arguments char concat @Description("Concatenates given character strings") // Given CHAR type max length, if the result type is valid, allocation cannot fail. @ScalarFunction(neverFails = true) From a9a535e4e77c11f9ce9bb71e2c5666334740deb2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 05:46:42 +0000 Subject: [PATCH 18/18] Bump com.azure:azure-core-tracing-opentelemetry Bumps [com.azure:azure-core-tracing-opentelemetry](https://github.com/Azure/azure-sdk-for-java) from 1.0.0-beta.65 to 1.0.0-beta.66. - [Release notes](https://github.com/Azure/azure-sdk-for-java/releases) - [Commits](https://github.com/Azure/azure-sdk-for-java/compare/azure-core-experimental_1.0.0-beta.65...com.azure+azure-core-experimental_1.0.0-beta.66) --- updated-dependencies: - dependency-name: com.azure:azure-core-tracing-opentelemetry dependency-version: 1.0.0-beta.66 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7cc56342ed3b..2d301cb4c581 100644 --- a/pom.xml +++ b/pom.xml @@ -349,7 +349,7 @@ com.azure azure-core-tracing-opentelemetry - 1.0.0-beta.65 + 1.0.0-beta.66 com.azure