columns) {
this(schemaId, columns, ImmutableSet.of());
}
@@ -629,4 +635,33 @@ public static void checkCompatibility(Schema schema, int formatVersion) {
formatVersion, Joiner.on("\n- ").join(problems.values())));
}
}
+
+ /**
+ * Indexes all fields from schemas.
+ *
+ * This method favors field definitions from higher schema IDs to handle type promotions.
+ *
+ * @param schemas the collection of schemas to index
+ * @return a map of field IDs to fields
+ */
+ public static Map indexFields(Collection schemas) {
+ if (schemas.size() == 1) {
+ Schema schema = Iterables.getOnlyElement(schemas);
+ return schema.lazyIdToField();
+ }
+
+ Map fields = Maps.newHashMap();
+
+ for (Schema schema : sortAndDeduplicate(schemas)) {
+ fields.putAll(schema.lazyIdToField());
+ }
+
+ return fields;
+ }
+
+ private static Set sortAndDeduplicate(Collection schemas) {
+ Set sortedSchemas = Sets.newTreeSet(Comparator.comparingInt(Schema::schemaId));
+ sortedSchemas.addAll(schemas);
+ return sortedSchemas;
+ }
}
diff --git a/api/src/main/java/org/apache/iceberg/Snapshot.java b/api/src/main/java/org/apache/iceberg/Snapshot.java
index 9bbb57ed4824..8a74dca6d053 100644
--- a/api/src/main/java/org/apache/iceberg/Snapshot.java
+++ b/api/src/main/java/org/apache/iceberg/Snapshot.java
@@ -112,7 +112,9 @@ public interface Snapshot extends Serializable {
*
* @param io a {@link FileIO} instance used for reading files from storage
* @return all data files added to the table in this snapshot.
+ * @deprecated will be removed in 2.0.0; use SnapshotChanges#builderFor(Table) instead
*/
+ @Deprecated
Iterable addedDataFiles(FileIO io);
/**
@@ -124,7 +126,9 @@ public interface Snapshot extends Serializable {
*
* @param io a {@link FileIO} instance used for reading files from storage
* @return all data files removed from the table in this snapshot.
+ * @deprecated will be removed in 2.0.0; use SnapshotChanges#builderFor(Table) instead
*/
+ @Deprecated
Iterable removedDataFiles(FileIO io);
/**
@@ -135,7 +139,9 @@ public interface Snapshot extends Serializable {
*
* @param io a {@link FileIO} instance used for reading files from storage
* @return all delete files added to the table in this snapshot
+ * @deprecated will be removed in 2.0.0; use SnapshotChanges#builderFor(Table) instead
*/
+ @Deprecated
default Iterable addedDeleteFiles(FileIO io) {
throw new UnsupportedOperationException(
this.getClass().getName() + " doesn't implement addedDeleteFiles");
@@ -149,7 +155,9 @@ default Iterable addedDeleteFiles(FileIO io) {
*
* @param io a {@link FileIO} instance used for reading files from storage
* @return all delete files removed from the table in this snapshot
+ * @deprecated will be removed in 2.0.0; use SnapshotChanges#builderFor(Table) instead
*/
+ @Deprecated
default Iterable removedDeleteFiles(FileIO io) {
throw new UnsupportedOperationException(
this.getClass().getName() + " doesn't implement removedDeleteFiles");
@@ -175,7 +183,7 @@ default Integer schemaId() {
/**
* The row-id of the first newly added row in this snapshot. All rows added in this snapshot will
* have a row-id assigned to them greater than this value. All rows with a row-id less than this
- * value were created in a snapshot that was added to the table (but not necessarily commited to
+ * value were created in a snapshot that was added to the table (but not necessarily committed to
* this branch) in the past.
*
* @return the first row-id to be used in this snapshot or null when row lineage is not supported
diff --git a/api/src/main/java/org/apache/iceberg/Table.java b/api/src/main/java/org/apache/iceberg/Table.java
index 97ea9ba76526..3c0689e89288 100644
--- a/api/src/main/java/org/apache/iceberg/Table.java
+++ b/api/src/main/java/org/apache/iceberg/Table.java
@@ -83,6 +83,18 @@ default IncrementalChangelogScan newIncrementalChangelogScan() {
throw new UnsupportedOperationException("Incremental changelog scan is not supported");
}
+ /**
+ * Create a new {@link PartitionStatisticsScan} for this table.
+ *
+ * Once a partition statistics scan is created, it can be refined to project columns and filter
+ * data.
+ *
+ * @return a partition statistics scan for this table
+ */
+ default PartitionStatisticsScan newPartitionStatisticsScan() {
+ throw new UnsupportedOperationException("Partition statistics scan is not supported");
+ }
+
/**
* Return the {@link Schema schema} for this table.
*
diff --git a/api/src/main/java/org/apache/iceberg/actions/DeleteOrphanFiles.java b/api/src/main/java/org/apache/iceberg/actions/DeleteOrphanFiles.java
index 4e8f80fa833f..ab12a3b7c1e3 100644
--- a/api/src/main/java/org/apache/iceberg/actions/DeleteOrphanFiles.java
+++ b/api/src/main/java/org/apache/iceberg/actions/DeleteOrphanFiles.java
@@ -142,6 +142,11 @@ default DeleteOrphanFiles equalAuthorities(Map newEqualAuthoriti
interface Result {
/** Returns locations of orphan files. */
Iterable orphanFileLocations();
+
+ /** Returns the total number of orphan files. */
+ default long orphanFilesCount() {
+ return 0;
+ }
}
/**
diff --git a/api/src/main/java/org/apache/iceberg/actions/RewriteTablePath.java b/api/src/main/java/org/apache/iceberg/actions/RewriteTablePath.java
index 4f65d46f57ef..8d823aa804d7 100644
--- a/api/src/main/java/org/apache/iceberg/actions/RewriteTablePath.java
+++ b/api/src/main/java/org/apache/iceberg/actions/RewriteTablePath.java
@@ -18,6 +18,8 @@
*/
package org.apache.iceberg.actions;
+import java.util.concurrent.ExecutorService;
+
/**
* An action that rewrites the table's metadata files to a staging directory, replacing all source
* prefixes in absolute paths with a specified target prefix. There are two modes:
@@ -99,6 +101,18 @@ default RewriteTablePath createFileList(boolean createFileList) {
return this;
}
+ /**
+ * Passes an alternative executor service that will be used for version file and manifest list
+ * rewriting. If this method is not called, these operations will be performed sequentially.
+ *
+ * @param executorService an executor service to parallelize metadata rewriting
+ * @return this for method chaining
+ */
+ default RewriteTablePath executeWith(ExecutorService executorService) {
+ throw new UnsupportedOperationException(
+ "This implementation does not support providing an ExecutorService.");
+ }
+
/** The action result that contains a summary of the execution. */
interface Result {
/** Staging location of rewritten files */
diff --git a/api/src/main/java/org/apache/iceberg/catalog/Catalog.java b/api/src/main/java/org/apache/iceberg/catalog/Catalog.java
index 897acd2e3ba6..3438f445ec2a 100644
--- a/api/src/main/java/org/apache/iceberg/catalog/Catalog.java
+++ b/api/src/main/java/org/apache/iceberg/catalog/Catalog.java
@@ -338,6 +338,8 @@ default void invalidateTable(TableIdentifier identifier) {}
/**
* Register a table with the catalog if it does not exist.
*
+ * For overwrite support, see {@link #registerTable(TableIdentifier, String, boolean)}.
+ *
* @param identifier a table identifier
* @param metadataFileLocation the location of a metadata file
* @return a Table instance
@@ -347,6 +349,25 @@ default Table registerTable(TableIdentifier identifier, String metadataFileLocat
throw new UnsupportedOperationException("Registering tables is not supported");
}
+ /**
+ * Register a table with the catalog.
+ *
+ * @param identifier a table identifier
+ * @param metadataFileLocation the location of a metadata file
+ * @param overwrite whether to overwrite an existing table registration
+ * @return a Table instance
+ * @throws AlreadyExistsException if {@code overwrite} is false and the table already exists in
+ * the catalog
+ */
+ default Table registerTable(
+ TableIdentifier identifier, String metadataFileLocation, boolean overwrite) {
+ if (!overwrite) {
+ return registerTable(identifier, metadataFileLocation);
+ }
+
+ throw new UnsupportedOperationException("Registering tables with overwrite is not supported");
+ }
+
/**
* Instantiate a builder to either create a table or start a create/replace transaction.
*
diff --git a/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java b/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java
index fe29f8918531..b54570485357 100644
--- a/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java
+++ b/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java
@@ -162,6 +162,9 @@ public Object wrappedIdentity() {
/**
* Register a table if it does not exist.
*
+ *
For overwrite support, see {@link #registerTable(SessionContext, TableIdentifier, String,
+ * boolean)}.
+ *
* @param context session context
* @param ident a table identifier
* @param metadataFileLocation the location of a metadata file
@@ -170,6 +173,29 @@ public Object wrappedIdentity() {
*/
Table registerTable(SessionContext context, TableIdentifier ident, String metadataFileLocation);
+ /**
+ * Register a table.
+ *
+ * @param context session context
+ * @param ident a table identifier
+ * @param metadataFileLocation the location of a metadata file
+ * @param overwrite whether to overwrite an existing table registration
+ * @return a Table instance
+ * @throws AlreadyExistsException if {@code overwrite} is false and the table already exists in
+ * the catalog
+ */
+ default Table registerTable(
+ SessionContext context,
+ TableIdentifier ident,
+ String metadataFileLocation,
+ boolean overwrite) {
+ if (!overwrite) {
+ return registerTable(context, ident, metadataFileLocation);
+ }
+
+ throw new UnsupportedOperationException("Registering tables with overwrite is not supported");
+ }
+
/**
* Check whether table exists.
*
diff --git a/api/src/main/java/org/apache/iceberg/catalog/TableIdentifier.java b/api/src/main/java/org/apache/iceberg/catalog/TableIdentifier.java
index cbb5dc8d8fd2..9b9fbdcbb0b7 100644
--- a/api/src/main/java/org/apache/iceberg/catalog/TableIdentifier.java
+++ b/api/src/main/java/org/apache/iceberg/catalog/TableIdentifier.java
@@ -80,7 +80,9 @@ public String name() {
public TableIdentifier toLowerCase() {
String[] newLevels =
- Arrays.stream(namespace().levels()).map(String::toLowerCase).toArray(String[]::new);
+ Arrays.stream(namespace().levels())
+ .map(s -> s.toLowerCase(Locale.ROOT))
+ .toArray(String[]::new);
String newName = name().toLowerCase(Locale.ROOT);
return TableIdentifier.of(Namespace.of(newLevels), newName);
}
diff --git a/api/src/main/java/org/apache/iceberg/catalog/ViewCatalog.java b/api/src/main/java/org/apache/iceberg/catalog/ViewCatalog.java
index ca470eec7171..52a715f807fa 100644
--- a/api/src/main/java/org/apache/iceberg/catalog/ViewCatalog.java
+++ b/api/src/main/java/org/apache/iceberg/catalog/ViewCatalog.java
@@ -106,6 +106,19 @@ default boolean viewExists(TableIdentifier identifier) {
*/
default void invalidateView(TableIdentifier identifier) {}
+ /**
+ * Register a view with the catalog if it does not exist.
+ *
+ * @param identifier a view identifier
+ * @param metadataFileLocation the location of a metadata file
+ * @return a View instance
+ * @throws AlreadyExistsException if a table/view with the same identifier already exists in the
+ * catalog.
+ */
+ default View registerView(TableIdentifier identifier, String metadataFileLocation) {
+ throw new UnsupportedOperationException("Registering views is not supported");
+ }
+
/**
* Initialize a view catalog given a custom name and a map of catalog properties.
*
diff --git a/api/src/main/java/org/apache/iceberg/catalog/ViewSessionCatalog.java b/api/src/main/java/org/apache/iceberg/catalog/ViewSessionCatalog.java
index 106e20d3bce1..0e195665f3c2 100644
--- a/api/src/main/java/org/apache/iceberg/catalog/ViewSessionCatalog.java
+++ b/api/src/main/java/org/apache/iceberg/catalog/ViewSessionCatalog.java
@@ -106,6 +106,21 @@ default boolean viewExists(SessionCatalog.SessionContext context, TableIdentifie
*/
default void invalidateView(SessionCatalog.SessionContext context, TableIdentifier identifier) {}
+ /**
+ * Register a view if it does not exist.
+ *
+ * @param context session context
+ * @param ident a view identifier
+ * @param metadataFileLocation the location of a metadata file
+ * @return a View instance
+ * @throws AlreadyExistsException if a table/view with the same identifier already exists in the
+ * catalog.
+ */
+ default View registerView(
+ SessionCatalog.SessionContext context, TableIdentifier ident, String metadataFileLocation) {
+ throw new UnsupportedOperationException("Registering views is not supported");
+ }
+
/**
* Initialize a view catalog given a custom name and a map of catalog properties.
*
diff --git a/api/src/main/java/org/apache/iceberg/encryption/EncryptedKey.java b/api/src/main/java/org/apache/iceberg/encryption/EncryptedKey.java
index c7e594efd147..c7e987e84b7d 100644
--- a/api/src/main/java/org/apache/iceberg/encryption/EncryptedKey.java
+++ b/api/src/main/java/org/apache/iceberg/encryption/EncryptedKey.java
@@ -18,10 +18,11 @@
*/
package org.apache.iceberg.encryption;
+import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.Map;
-public interface EncryptedKey {
+public interface EncryptedKey extends Serializable {
String keyId();
ByteBuffer encryptedKeyMetadata();
diff --git a/api/src/main/java/org/apache/iceberg/encryption/EncryptingFileIO.java b/api/src/main/java/org/apache/iceberg/encryption/EncryptingFileIO.java
index 2ab335a4d6c6..c19da0f01e5b 100644
--- a/api/src/main/java/org/apache/iceberg/encryption/EncryptingFileIO.java
+++ b/api/src/main/java/org/apache/iceberg/encryption/EncryptingFileIO.java
@@ -30,8 +30,10 @@
import org.apache.iceberg.ManifestFile;
import org.apache.iceberg.ManifestListFile;
import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.io.FileInfo;
import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.io.SupportsPrefixOperations;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
@@ -46,7 +48,11 @@ public static EncryptingFileIO combine(FileIO io, EncryptionManager em) {
return combine(encryptingIO.io, em);
}
- return new EncryptingFileIO(io, em);
+ if (io instanceof SupportsPrefixOperations) {
+ return new WithSupportsPrefixOperations((SupportsPrefixOperations) io, em);
+ } else {
+ return new EncryptingFileIO(io, em);
+ }
}
private final FileIO io;
@@ -142,6 +148,11 @@ public void deleteFile(String path) {
io.deleteFile(path);
}
+ @Override
+ public Map properties() {
+ return io.properties();
+ }
+
@Override
public void close() {
io.close();
@@ -206,4 +217,25 @@ public EncryptionKeyMetadata copy() {
return new SimpleKeyMetadata(metadataBuffer.duplicate());
}
}
+
+ static class WithSupportsPrefixOperations extends EncryptingFileIO
+ implements SupportsPrefixOperations {
+
+ private final SupportsPrefixOperations prefixIo;
+
+ WithSupportsPrefixOperations(SupportsPrefixOperations io, EncryptionManager em) {
+ super(io, em);
+ this.prefixIo = io;
+ }
+
+ @Override
+ public Iterable listPrefix(String prefix) {
+ return prefixIo.listPrefix(prefix);
+ }
+
+ @Override
+ public void deletePrefix(String prefix) {
+ prefixIo.deletePrefix(prefix);
+ }
+ }
}
diff --git a/api/src/main/java/org/apache/iceberg/exceptions/NoSuchWarehouseException.java b/api/src/main/java/org/apache/iceberg/exceptions/NoSuchWarehouseException.java
new file mode 100644
index 000000000000..94ae50cd1c25
--- /dev/null
+++ b/api/src/main/java/org/apache/iceberg/exceptions/NoSuchWarehouseException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.exceptions;
+
+import com.google.errorprone.annotations.FormatMethod;
+
+/** Exception raised when attempting to load a warehouse that does not exist. */
+public class NoSuchWarehouseException extends RuntimeException {
+ @FormatMethod
+ public NoSuchWarehouseException(String message, Object... args) {
+ super(String.format(message, args));
+ }
+
+ @FormatMethod
+ public NoSuchWarehouseException(Throwable cause, String message, Object... args) {
+ super(String.format(message, args), cause);
+ }
+}
diff --git a/api/src/main/java/org/apache/iceberg/expressions/BoundAggregate.java b/api/src/main/java/org/apache/iceberg/expressions/BoundAggregate.java
index 03c371df25df..72f611df201d 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/BoundAggregate.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/BoundAggregate.java
@@ -51,6 +51,11 @@ Aggregator newAggregator() {
this.getClass().getName() + " does not implement newAggregator()");
}
+ boolean containsNan(DataFile file, int fieldId) {
+ Long nanCount = safeGet(file.nanValueCounts(), fieldId);
+ return nanCount != null && nanCount > 0;
+ }
+
@Override
public BoundReference ref() {
return term().ref();
diff --git a/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java b/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java
index d3dc00d914c7..af24ce40cac8 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java
@@ -37,7 +37,6 @@
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.transforms.Transforms;
-import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.DateTimeUtil;
import org.apache.iceberg.variants.PhysicalType;
@@ -69,7 +68,6 @@ public class ExpressionUtil {
"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}(:\\d{2}(.\\d{7,9})?)?([-+]\\d{2}:\\d{2}|Z)");
static final int LONG_IN_PREDICATE_ABBREVIATION_THRESHOLD = 10;
- private static final int LONG_IN_PREDICATE_ABBREVIATION_MIN_GAIN = 5;
private ExpressionUtil() {}
@@ -148,21 +146,28 @@ public static String toSanitizedString(
}
/**
- * Extracts an expression that references only the given column IDs from the given expression.
+ * Returns an expression that retains only predicates which reference one of the given field IDs.
*
- * The result is inclusive. If a row would match the original filter, it must match the result
- * filter.
- *
- * @param expression a filter Expression
- * @param schema a Schema
+ * @param expression a filter expression
+ * @param schema schema for binding references
* @param caseSensitive whether binding is case sensitive
- * @param ids field IDs used to match predicates to extract from the expression
- * @return an Expression that selects at least the same rows as the original using only the IDs
+ * @param ids field IDs to retain predicates for
+ * @return expression containing only predicates that reference the given IDs
*/
public static Expression extractByIdInclusive(
Expression expression, Schema schema, boolean caseSensitive, int... ids) {
- PartitionSpec spec = identitySpec(schema, ids);
- return Projections.inclusive(spec, caseSensitive).project(Expressions.rewriteNot(expression));
+ if (ids == null || ids.length == 0) {
+ return Expressions.alwaysTrue();
+ }
+
+ ImmutableSet.Builder retainIds = ImmutableSet.builder();
+ for (int id : ids) {
+ retainIds.add(id);
+ }
+
+ return ExpressionVisitors.visit(
+ Expressions.rewriteNot(expression),
+ new RetainPredicatesByFieldIdVisitor(schema, caseSensitive, retainIds.build()));
}
/**
@@ -212,6 +217,10 @@ public static boolean selectsPartitions(Expression expr, Table table, boolean ca
*/
public static boolean selectsPartitions(
Expression expr, PartitionSpec spec, boolean caseSensitive) {
+ if (spec.isUnpartitioned()) {
+ return false;
+ }
+
return equivalent(
Projections.inclusive(spec, caseSensitive).project(expr),
Projections.strict(spec, caseSensitive).project(expr),
@@ -261,6 +270,61 @@ public static UnboundTerm unbind(Term term) {
throw new UnsupportedOperationException("Cannot unbind unsupported term: " + term);
}
+ private static class RetainPredicatesByFieldIdVisitor
+ extends ExpressionVisitors.ExpressionVisitor {
+ private final Schema schema;
+ private final boolean caseSensitive;
+ private final Set retainFieldIds;
+
+ RetainPredicatesByFieldIdVisitor(
+ Schema schema, boolean caseSensitive, Set retainFieldIds) {
+ this.schema = schema;
+ this.caseSensitive = caseSensitive;
+ this.retainFieldIds = retainFieldIds;
+ }
+
+ @Override
+ public Expression alwaysTrue() {
+ return Expressions.alwaysTrue();
+ }
+
+ @Override
+ public Expression alwaysFalse() {
+ return Expressions.alwaysFalse();
+ }
+
+ @Override
+ public Expression not(Expression result) {
+ return Expressions.not(result);
+ }
+
+ @Override
+ public Expression and(Expression leftResult, Expression rightResult) {
+ return Expressions.and(leftResult, rightResult);
+ }
+
+ @Override
+ public Expression or(Expression leftResult, Expression rightResult) {
+ return Expressions.or(leftResult, rightResult);
+ }
+
+ @Override
+ public Expression predicate(BoundPredicate pred) {
+ return retainFieldIds.contains(pred.ref().fieldId()) ? pred : Expressions.alwaysTrue();
+ }
+
+ @Override
+ public Expression predicate(UnboundPredicate pred) {
+ Expression bound = Binder.bind(schema.asStruct(), pred, caseSensitive);
+ if (bound instanceof BoundPredicate) {
+ return retainFieldIds.contains(((BoundPredicate) bound).ref().fieldId())
+ ? pred
+ : Expressions.alwaysTrue();
+ }
+ return Expressions.alwaysTrue();
+ }
+ }
+
private static class ExpressionSanitizer
extends ExpressionVisitors.ExpressionVisitor {
private final long now;
@@ -307,15 +371,13 @@ public Expression predicate(BoundPredicate pred) {
} else if (pred.isLiteralPredicate()) {
BoundLiteralPredicate bound = (BoundLiteralPredicate) pred;
return new UnboundPredicate<>(
- pred.op(),
- unbind(pred.term()),
- (T) sanitize(bound.term().type(), bound.literal(), now, today));
+ pred.op(), unbind(pred.term()), (T) sanitize(bound.literal(), now, today));
} else if (pred.isSetPredicate()) {
BoundSetPredicate bound = (BoundSetPredicate) pred;
Iterable iter =
() ->
bound.literalSet().stream()
- .map(lit -> (T) sanitize(bound.term().type(), lit, now, today))
+ .map(lit -> (T) sanitize((Literal) lit, now, today))
.iterator();
return new UnboundPredicate<>(pred.op(), unbind(pred.term()), iter);
}
@@ -392,7 +454,7 @@ public String or(String leftResult, String rightResult) {
}
private String value(BoundLiteralPredicate pred) {
- return sanitize(pred.term().type(), pred.literal().value(), nowMicros, today);
+ return sanitize(pred.literal(), nowMicros, today);
}
@Override
@@ -424,7 +486,7 @@ public String predicate(BoundPredicate pred) {
+ " IN "
+ abbreviateValues(
pred.asSetPredicate().literalSet().stream()
- .map(lit -> sanitize(pred.term().type(), lit, nowMicros, today))
+ .map(lit -> sanitize((Literal) lit, nowMicros, today))
.collect(Collectors.toList()))
.stream()
.collect(Collectors.joining(", ", "(", ")"));
@@ -433,7 +495,7 @@ public String predicate(BoundPredicate pred) {
+ " NOT IN "
+ abbreviateValues(
pred.asSetPredicate().literalSet().stream()
- .map(lit -> sanitize(pred.term().type(), lit, nowMicros, today))
+ .map(lit -> sanitize((Literal) lit, nowMicros, today))
.collect(Collectors.toList()))
.stream()
.collect(Collectors.joining(", ", "(", ")"));
@@ -502,61 +564,24 @@ public String predicate(UnboundPredicate pred) {
private static List abbreviateValues(List sanitizedValues) {
if (sanitizedValues.size() >= LONG_IN_PREDICATE_ABBREVIATION_THRESHOLD) {
- Set distinctValues = ImmutableSet.copyOf(sanitizedValues);
- if (distinctValues.size()
- <= sanitizedValues.size() - LONG_IN_PREDICATE_ABBREVIATION_MIN_GAIN) {
- List abbreviatedList = Lists.newArrayListWithCapacity(distinctValues.size() + 1);
- abbreviatedList.addAll(distinctValues);
+ List distinctValues = ImmutableSet.copyOf(sanitizedValues).asList();
+ int abbreviatedSize =
+ Math.min(distinctValues.size(), LONG_IN_PREDICATE_ABBREVIATION_THRESHOLD);
+ List abbreviatedList = Lists.newArrayListWithCapacity(abbreviatedSize + 1);
+ abbreviatedList.addAll(distinctValues.subList(0, abbreviatedSize));
+ if (abbreviatedSize < sanitizedValues.size()) {
abbreviatedList.add(
String.format(
Locale.ROOT,
"... (%d values hidden, %d in total)",
- sanitizedValues.size() - distinctValues.size(),
+ sanitizedValues.size() - abbreviatedSize,
sanitizedValues.size()));
- return abbreviatedList;
}
+ return abbreviatedList;
}
return sanitizedValues;
}
- private static String sanitize(Type type, Literal lit, long now, int today) {
- return sanitize(type, lit.value(), now, today);
- }
-
- private static String sanitize(Type type, Object value, long now, int today) {
- switch (type.typeId()) {
- case INTEGER:
- case LONG:
- return sanitizeNumber((Number) value, "int");
- case FLOAT:
- case DOUBLE:
- return sanitizeNumber((Number) value, "float");
- case DATE:
- return sanitizeDate((int) value, today);
- case TIME:
- return "(time)";
- case TIMESTAMP:
- return sanitizeTimestamp((long) value, now);
- case TIMESTAMP_NANO:
- return sanitizeTimestamp(DateTimeUtil.nanosToMicros((long) value / 1000), now);
- case STRING:
- return sanitizeString((CharSequence) value, now, today);
- case VARIANT:
- return sanitizeVariant((Variant) value, now, today);
- case UNKNOWN:
- return "(unknown)";
- case BOOLEAN:
- case UUID:
- case DECIMAL:
- case FIXED:
- case BINARY:
- // for boolean, uuid, decimal, fixed, unknown, and binary, match the string result
- return sanitizeSimpleString(value.toString());
- }
- throw new UnsupportedOperationException(
- String.format("Cannot sanitize value for unsupported type %s: %s", type, value));
- }
-
private static String sanitize(Literal literal, long now, int today) {
if (literal instanceof Literals.StringLiteral) {
return sanitizeString(((Literals.StringLiteral) literal).value(), now, today);
@@ -735,14 +760,4 @@ private static String sanitizeVariantValue(
}
return builder.toString();
}
-
- private static PartitionSpec identitySpec(Schema schema, int... ids) {
- PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema);
-
- for (int id : ids) {
- specBuilder.identity(schema.findColumnName(id));
- }
-
- return specBuilder.build();
- }
}
diff --git a/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java b/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java
index aa0441f49011..81cbbe785519 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java
@@ -327,6 +327,14 @@ public Boolean eq(Bound term, Literal lit) {
public Boolean notEq(Bound term, Literal lit) {
// because the bounds are not necessarily a min or max value, this cannot be answered using
// them. notEq(col, X) with (X, Y) doesn't guarantee that X is a value in col.
+ // However, when min == max and the file has no nulls or NaN values, we can safely prune
+ // if that value equals the literal.
+ T value = uniqueValue(term);
+
+ if (value != null && lit.comparator().compare(value, lit.value()) == 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
return ROWS_MIGHT_MATCH;
}
@@ -381,6 +389,14 @@ public Boolean in(Bound term, Set literalSet) {
public Boolean notIn(Bound term, Set literalSet) {
// because the bounds are not necessarily a min or max value, this cannot be answered using
// them. notIn(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value in col.
+ // However, when min == max and the file has no nulls or NaN values, we can safely prune
+ // if that value is in the exclusion set.
+ T value = uniqueValue(term);
+
+ if (value != null && literalSet.contains(value)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
return ROWS_MIGHT_MATCH;
}
@@ -490,6 +506,34 @@ private boolean containsNaNsOnly(Integer id) {
&& nanCounts.get(id).equals(valueCounts.get(id));
}
+ /**
+ * Returns the column's single value if all rows contain the same value. Defined as a column
+ * with no nulls, no NaNs, and lower bound equals upper bound. Returns null otherwise.
+ */
+ private T uniqueValue(Bound term) {
+ int id = term.ref().fieldId();
+ if (mayContainNull(id)) {
+ return null;
+ }
+
+ T lower = lowerBound(term);
+ T upper = upperBound(term);
+
+ if (lower == null || upper == null || NaNUtil.isNaN(lower) || NaNUtil.isNaN(upper)) {
+ return null;
+ }
+
+ if (nanCounts != null && nanCounts.containsKey(id) && nanCounts.get(id) != 0) {
+ return null;
+ }
+
+ if (!lower.equals(upper)) {
+ return null;
+ }
+
+ return lower;
+ }
+
private T lowerBound(Bound term) {
if (term instanceof BoundReference) {
return parseLowerBound((BoundReference) term);
diff --git a/api/src/main/java/org/apache/iceberg/expressions/Literals.java b/api/src/main/java/org/apache/iceberg/expressions/Literals.java
index 3a45eb804f35..c54dff72f87e 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/Literals.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/Literals.java
@@ -30,6 +30,7 @@
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.Comparator;
+import java.util.Locale;
import java.util.Objects;
import java.util.UUID;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
@@ -47,6 +48,7 @@ class Literals {
private Literals() {}
private static final OffsetDateTime EPOCH = Instant.ofEpochSecond(0).atOffset(ZoneOffset.UTC);
+ private static final BaseEncoding BASE16_ENCODING = BaseEncoding.base16();
private static final LocalDate EPOCH_DAY = EPOCH.toLocalDate();
/**
@@ -587,6 +589,32 @@ public Literal to(Type type) {
BigDecimal decimal = new BigDecimal(value().toString());
return (Literal) new DecimalLiteral(decimal);
+ case FIXED:
+ try {
+ ByteBuffer buffer =
+ ByteBuffer.wrap(
+ BASE16_ENCODING.decode(value().toString().toUpperCase(Locale.ROOT)));
+ Types.FixedType fixed = (Types.FixedType) type;
+ if (buffer.remaining() == fixed.length()) {
+ return (Literal) new FixedLiteral(buffer);
+ }
+ return null;
+ } catch (IllegalArgumentException e) {
+ // Invalid hex string
+ return null;
+ }
+
+ case BINARY:
+ try {
+ return (Literal)
+ new BinaryLiteral(
+ ByteBuffer.wrap(
+ BASE16_ENCODING.decode(value().toString().toUpperCase(Locale.ROOT))));
+ } catch (IllegalArgumentException e) {
+ // Invalid hex string
+ return null;
+ }
+
default:
return null;
}
@@ -670,7 +698,7 @@ Object writeReplace() throws ObjectStreamException {
@Override
public String toString() {
byte[] bytes = ByteBuffers.toByteArray(value());
- return "X'" + BaseEncoding.base16().encode(bytes) + "'";
+ return "X'" + BASE16_ENCODING.encode(bytes) + "'";
}
}
@@ -716,7 +744,7 @@ protected Type.TypeID typeId() {
@Override
public String toString() {
byte[] bytes = ByteBuffers.toByteArray(value());
- return "X'" + BaseEncoding.base16().encode(bytes) + "'";
+ return "X'" + BASE16_ENCODING.encode(bytes) + "'";
}
}
}
diff --git a/api/src/main/java/org/apache/iceberg/expressions/ManifestEvaluator.java b/api/src/main/java/org/apache/iceberg/expressions/ManifestEvaluator.java
index fc3d394203ff..21762946ca33 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/ManifestEvaluator.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/ManifestEvaluator.java
@@ -269,6 +269,14 @@ public Boolean eq(BoundReference ref, Literal lit) {
public Boolean notEq(BoundReference ref, Literal lit) {
// because the bounds are not necessarily a min or max value, this cannot be answered using
// them. notEq(col, X) with (X, Y) doesn't guarantee that X is a value in col.
+ // However, when lower == upper and the manifest has no nulls or NaN values, we can safely
+ // prune if that value equals the literal.
+ T value = uniqueValue(ref);
+
+ if (value != null && lit.comparator().compare(value, lit.value()) == 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
return ROWS_MIGHT_MATCH;
}
@@ -313,6 +321,14 @@ public Boolean in(BoundReference ref, Set literalSet) {
public Boolean notIn(BoundReference ref, Set literalSet) {
// because the bounds are not necessarily a min or max value, this cannot be answered using
// them. notIn(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value in col.
+ // However, when lower == upper and the manifest has no nulls or NaN values, we can safely
+ // prune if that value is in the exclusion set.
+ T value = uniqueValue(ref);
+
+ if (value != null && literalSet.contains(value)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
return ROWS_MIGHT_MATCH;
}
@@ -400,6 +416,43 @@ public Boolean notStartsWith(BoundReference ref, Literal lit) {
return ROWS_MIGHT_MATCH;
}
+ /**
+ * Returns the partition field's single value if all partitions contain the same value. Defined
+ * as a partition field with no nulls, no NaNs (for floating-point types), and lower bound
+ * equals upper bound. Returns null otherwise.
+ */
+ private T uniqueValue(BoundReference ref) {
+ int pos = Accessors.toPosition(ref.accessor());
+ PartitionFieldSummary fieldStats = stats.get(pos);
+
+ if (fieldStats.containsNull()) {
+ return null;
+ }
+
+ Type.TypeID typeId = ref.type().typeId();
+ if (Type.TypeID.FLOAT.equals(typeId) || Type.TypeID.DOUBLE.equals(typeId)) {
+ if (fieldStats.containsNaN() == null || fieldStats.containsNaN()) {
+ return null;
+ }
+ }
+
+ ByteBuffer lowerBound = fieldStats.lowerBound();
+ ByteBuffer upperBound = fieldStats.upperBound();
+
+ if (lowerBound == null || upperBound == null) {
+ return null;
+ }
+
+ T lower = Conversions.fromByteBuffer(ref.type(), lowerBound);
+ T upper = Conversions.fromByteBuffer(ref.type(), upperBound);
+
+ if (ref.comparator().compare(lower, upper) != 0) {
+ return null;
+ }
+
+ return lower;
+ }
+
private boolean allValuesAreNull(PartitionFieldSummary summary, Type.TypeID typeId) {
// containsNull encodes whether at least one partition value is null,
// lowerBound is null if all partition values are null
diff --git a/api/src/main/java/org/apache/iceberg/expressions/MaxAggregate.java b/api/src/main/java/org/apache/iceberg/expressions/MaxAggregate.java
index d37af7470df2..2948ffa421ae 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/MaxAggregate.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/MaxAggregate.java
@@ -40,6 +40,10 @@ protected MaxAggregate(BoundTerm term) {
@Override
protected boolean hasValue(DataFile file) {
+ // Can't determine max from metadata when NaN values are present since it could be -NaN or +NaN
+ if (containsNan(file, fieldId)) {
+ return false;
+ }
boolean hasBound = safeContainsKey(file.upperBounds(), fieldId);
Long valueCount = safeGet(file.valueCounts(), fieldId);
Long nullCount = safeGet(file.nullValueCounts(), fieldId);
diff --git a/api/src/main/java/org/apache/iceberg/expressions/MinAggregate.java b/api/src/main/java/org/apache/iceberg/expressions/MinAggregate.java
index 667b66d6500d..cf13f9256228 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/MinAggregate.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/MinAggregate.java
@@ -40,6 +40,10 @@ protected MinAggregate(BoundTerm term) {
@Override
protected boolean hasValue(DataFile file) {
+ // Can't determine min from metadata when NaN values are present since it could be -NaN or +NaN
+ if (containsNan(file, fieldId)) {
+ return false;
+ }
boolean hasBound = safeContainsKey(file.lowerBounds(), fieldId);
Long valueCount = safeGet(file.valueCounts(), fieldId);
Long nullCount = safeGet(file.nullValueCounts(), fieldId);
diff --git a/api/src/main/java/org/apache/iceberg/expressions/StrictMetricsEvaluator.java b/api/src/main/java/org/apache/iceberg/expressions/StrictMetricsEvaluator.java
index c225f21da8a8..f57ba8bc2793 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/StrictMetricsEvaluator.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/StrictMetricsEvaluator.java
@@ -22,6 +22,7 @@
import java.nio.ByteBuffer;
import java.util.Collection;
+import java.util.Comparator;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
@@ -29,6 +30,7 @@
import org.apache.iceberg.DataFile;
import org.apache.iceberg.Schema;
import org.apache.iceberg.expressions.ExpressionVisitors.BoundExpressionVisitor;
+import org.apache.iceberg.types.Comparators;
import org.apache.iceberg.types.Conversions;
import org.apache.iceberg.types.Types.StructType;
import org.apache.iceberg.util.NaNUtil;
@@ -462,13 +464,77 @@ public Boolean notIn(BoundReference ref, Set literalSet) {
@Override
public Boolean startsWith(BoundReference ref, Literal lit) {
+ int id = ref.fieldId();
+ if (isNestedColumn(id)) {
+ return ROWS_MIGHT_NOT_MATCH;
+ }
+
+ if (canContainNulls(id)) {
+ return ROWS_MIGHT_NOT_MATCH;
+ }
+
+ if (lowerBounds != null
+ && lowerBounds.containsKey(id)
+ && upperBounds != null
+ && upperBounds.containsKey(id)) {
+ String prefix = (String) lit.value();
+ Comparator comparator = Comparators.charSequences();
+ CharSequence lower = Conversions.fromByteBuffer(ref.type(), lowerBounds.get(id));
+ CharSequence upper = Conversions.fromByteBuffer(ref.type(), upperBounds.get(id));
+
+ // if lower is shorter than the prefix then lower doesn't start with the prefix
+ if (lower.length() < prefix.length()) {
+ return ROWS_MIGHT_NOT_MATCH;
+ }
+
+ if (comparator.compare(lower.subSequence(0, prefix.length()), prefix) == 0) {
+ // if upper is shorter than the prefix then upper can't start with the prefix
+ if (upper.length() < prefix.length()) {
+ return ROWS_MIGHT_NOT_MATCH;
+ }
+
+ if (comparator.compare(upper.subSequence(0, prefix.length()), prefix) == 0) {
+ // both bounds start with the prefix, so all rows must start with the prefix
+ return ROWS_MUST_MATCH;
+ }
+ }
+ }
+
return ROWS_MIGHT_NOT_MATCH;
}
@Override
public Boolean notStartsWith(BoundReference ref, Literal lit) {
- // TODO: Handle cases that definitely cannot match, such as notStartsWith("x") when the bounds
- // are ["a", "b"].
+ int id = ref.fieldId();
+ if (isNestedColumn(id)) {
+ return ROWS_MIGHT_NOT_MATCH;
+ }
+
+ if (containsNullsOnly(id)) {
+ return ROWS_MUST_MATCH;
+ }
+
+ String prefix = (String) lit.value();
+ Comparator comparator = Comparators.charSequences();
+
+ if (lowerBounds != null && lowerBounds.containsKey(id)) {
+ CharSequence lower = Conversions.fromByteBuffer(ref.type(), lowerBounds.get(id));
+ // truncate lower bound so that its length is not greater than the length of prefix
+ int length = Math.min(prefix.length(), lower.length());
+ if (comparator.compare(lower.subSequence(0, length), prefix) > 0) {
+ return ROWS_MUST_MATCH;
+ }
+ }
+
+ if (upperBounds != null && upperBounds.containsKey(id)) {
+ CharSequence upper = Conversions.fromByteBuffer(ref.type(), upperBounds.get(id));
+ // truncate upper bound so that its length is not greater than the length of prefix
+ int length = Math.min(prefix.length(), upper.length());
+ if (comparator.compare(upper.subSequence(0, length), prefix) < 0) {
+ return ROWS_MUST_MATCH;
+ }
+ }
+
return ROWS_MIGHT_NOT_MATCH;
}
diff --git a/api/src/main/java/org/apache/iceberg/expressions/VariantExpressionUtil.java b/api/src/main/java/org/apache/iceberg/expressions/VariantExpressionUtil.java
index dca11d5e4662..daec5216f0ef 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/VariantExpressionUtil.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/VariantExpressionUtil.java
@@ -24,13 +24,11 @@
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.DateTimeUtil;
import org.apache.iceberg.variants.PhysicalType;
import org.apache.iceberg.variants.VariantValue;
class VariantExpressionUtil {
- // TODO: Implement PhysicalType.TIME
- // TODO: Implement PhysicalType.TIMESTAMPNTZ_NANO and PhysicalType.TIMESTAMPTZ_NANO
- // TODO: Implement PhysicalType.UUID
private static final Map NO_CONVERSION_NEEDED =
ImmutableMap.builder()
.put(Types.IntegerType.get(), PhysicalType.INT32)
@@ -40,6 +38,10 @@ class VariantExpressionUtil {
.put(Types.DateType.get(), PhysicalType.DATE)
.put(Types.TimestampType.withoutZone(), PhysicalType.TIMESTAMPNTZ)
.put(Types.TimestampType.withZone(), PhysicalType.TIMESTAMPTZ)
+ .put(Types.TimestampNanoType.withoutZone(), PhysicalType.TIMESTAMPNTZ_NANOS)
+ .put(Types.TimestampNanoType.withZone(), PhysicalType.TIMESTAMPTZ_NANOS)
+ .put(Types.TimeType.get(), PhysicalType.TIME)
+ .put(Types.UUIDType.get(), PhysicalType.UUID)
.put(Types.StringType.get(), PhysicalType.STRING)
.put(Types.BinaryType.get(), PhysicalType.BINARY)
.put(Types.UnknownType.get(), PhysicalType.NULL)
@@ -47,7 +49,7 @@ class VariantExpressionUtil {
private VariantExpressionUtil() {}
- @SuppressWarnings("unchecked")
+ @SuppressWarnings({"unchecked", "CyclomaticComplexity"})
static T castTo(VariantValue value, Type type) {
if (value == null) {
return null;
@@ -111,6 +113,40 @@ static T castTo(VariantValue value, Type type) {
}
break;
+ case TIMESTAMP:
+ if (value.type() == PhysicalType.TIMESTAMPTZ_NANOS
+ || value.type() == PhysicalType.TIMESTAMPNTZ_NANOS) {
+ return (T)
+ (Long) DateTimeUtil.nanosToMicros(((Number) value.asPrimitive().get()).longValue());
+ } else if (value.type() == PhysicalType.DATE) {
+ return (T)
+ (Long)
+ DateTimeUtil.microsFromTimestamp(
+ DateTimeUtil.dateFromDays(((Number) value.asPrimitive().get()).intValue())
+ .atStartOfDay());
+ }
+ break;
+ case TIMESTAMP_NANO:
+ if (value.type() == PhysicalType.TIMESTAMPTZ || value.type() == PhysicalType.TIMESTAMPNTZ) {
+ return (T)
+ (Long) DateTimeUtil.microsToNanos(((Number) value.asPrimitive().get()).longValue());
+ } else if (value.type() == PhysicalType.DATE) {
+ return (T)
+ (Long)
+ DateTimeUtil.nanosFromTimestamp(
+ DateTimeUtil.dateFromDays(((Number) value.asPrimitive().get()).intValue())
+ .atStartOfDay());
+ }
+ break;
+ case DATE:
+ if (value.type() == PhysicalType.TIMESTAMPTZ || value.type() == PhysicalType.TIMESTAMPNTZ) {
+ return (T)
+ (Integer) DateTimeUtil.microsToDays(((Number) value.asPrimitive().get()).longValue());
+ } else if (value.type() == PhysicalType.TIMESTAMPTZ_NANOS
+ || value.type() == PhysicalType.TIMESTAMPNTZ_NANOS) {
+ return (T)
+ (Integer) DateTimeUtil.nanosToDays(((Number) value.asPrimitive().get()).longValue());
+ }
}
return null;
diff --git a/api/src/main/java/org/apache/iceberg/io/FileRange.java b/api/src/main/java/org/apache/iceberg/io/FileRange.java
index f6d5d9b41cca..78acf6374e42 100644
--- a/api/src/main/java/org/apache/iceberg/io/FileRange.java
+++ b/api/src/main/java/org/apache/iceberg/io/FileRange.java
@@ -18,7 +18,6 @@
*/
package org.apache.iceberg.io;
-import java.io.EOFException;
import java.nio.ByteBuffer;
import java.util.concurrent.CompletableFuture;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
@@ -28,13 +27,10 @@ public class FileRange {
private final long offset;
private final int length;
- public FileRange(CompletableFuture byteBuffer, long offset, int length)
- throws EOFException {
+ public FileRange(CompletableFuture byteBuffer, long offset, int length) {
Preconditions.checkNotNull(byteBuffer, "byteBuffer can't be null");
- Preconditions.checkArgument(
- length() >= 0, "Invalid length: %s in range (must be >= 0)", length);
- Preconditions.checkArgument(
- offset() >= 0, "Invalid offset: %s in range (must be >= 0)", offset);
+ Preconditions.checkArgument(length >= 0, "Invalid length: %s in range (must be >= 0)", length);
+ Preconditions.checkArgument(offset >= 0, "Invalid offset: %s in range (must be >= 0)", offset);
this.byteBuffer = byteBuffer;
this.offset = offset;
diff --git a/api/src/main/java/org/apache/iceberg/stats/FieldStatistic.java b/api/src/main/java/org/apache/iceberg/stats/FieldStatistic.java
deleted file mode 100644
index 7715359ea245..000000000000
--- a/api/src/main/java/org/apache/iceberg/stats/FieldStatistic.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iceberg.stats;
-
-import static org.apache.iceberg.types.Types.NestedField.optional;
-
-import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.Types;
-
-public enum FieldStatistic {
- VALUE_COUNT(0, "value_count"),
- NULL_VALUE_COUNT(1, "null_value_count"),
- NAN_VALUE_COUNT(2, "nan_value_count"),
- AVG_VALUE_SIZE(3, "avg_value_size"),
- MAX_VALUE_SIZE(4, "max_value_size"),
- LOWER_BOUND(5, "lower_bound"),
- UPPER_BOUND(6, "upper_bound"),
- EXACT_BOUNDS(7, "exact_bounds");
-
- private final int offset;
- private final String fieldName;
-
- FieldStatistic(int offset, String fieldName) {
- this.offset = offset;
- this.fieldName = fieldName;
- }
-
- public int offset() {
- return offset;
- }
-
- public String fieldName() {
- return fieldName;
- }
-
- public static FieldStatistic fromOffset(int offset) {
- switch (offset) {
- case 0:
- return VALUE_COUNT;
- case 1:
- return NULL_VALUE_COUNT;
- case 2:
- return NAN_VALUE_COUNT;
- case 3:
- return AVG_VALUE_SIZE;
- case 4:
- return MAX_VALUE_SIZE;
- case 5:
- return LOWER_BOUND;
- case 6:
- return UPPER_BOUND;
- case 7:
- return EXACT_BOUNDS;
- default:
- throw new IllegalArgumentException("Invalid statistic offset: " + offset);
- }
- }
-
- public static Types.StructType fieldStatsFor(Type type, int fieldId) {
- return Types.StructType.of(
- optional(
- fieldId + VALUE_COUNT.offset(),
- VALUE_COUNT.fieldName(),
- Types.LongType.get(),
- "Total value count, including null and NaN"),
- optional(
- fieldId + NULL_VALUE_COUNT.offset(),
- NULL_VALUE_COUNT.fieldName(),
- Types.LongType.get(),
- "Total null value count"),
- optional(
- fieldId + NAN_VALUE_COUNT.offset(),
- NAN_VALUE_COUNT.fieldName(),
- Types.LongType.get(),
- "Total NaN value count"),
- optional(
- fieldId + AVG_VALUE_SIZE.offset(),
- AVG_VALUE_SIZE.fieldName(),
- Types.IntegerType.get(),
- "Avg value size of variable-length types (String, Binary)"),
- optional(
- fieldId + MAX_VALUE_SIZE.offset(),
- MAX_VALUE_SIZE.fieldName(),
- Types.IntegerType.get(),
- "Max value size of variable-length types (String, Binary)"),
- optional(fieldId + LOWER_BOUND.offset(), LOWER_BOUND.fieldName(), type, "Lower bound"),
- optional(fieldId + UPPER_BOUND.offset(), UPPER_BOUND.fieldName(), type, "Upper bound"),
- optional(
- fieldId + EXACT_BOUNDS.offset(),
- EXACT_BOUNDS.fieldName(),
- Types.BooleanType.get(),
- "Whether the upper/lower bound is exact or not"));
- }
-}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/Transforms.java b/api/src/main/java/org/apache/iceberg/transforms/Transforms.java
index aacd4d430069..a3a6a3f6321d 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/Transforms.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/Transforms.java
@@ -29,7 +29,7 @@
* Factory methods for transforms.
*
* Most users should create transforms using a {@link PartitionSpec#builderFor(Schema)} partition
- * spec builder}.
+ * spec builder.
*
* @see PartitionSpec#builderFor(Schema) The partition spec builder.
*/
diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java
index b1c556be0667..22e461a39ef9 100644
--- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java
+++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java
@@ -19,6 +19,7 @@
package org.apache.iceberg.types;
import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -102,6 +103,19 @@ public static Schema select(Schema schema, Set fieldIds) {
return new Schema(ImmutableList.of(), schema.getAliases());
}
+ /**
+ * Selects fields from a schema by ID and returns them ordered by field ID.
+ *
+ * Unlike {@link #select(Schema, Set)}, which preserves the field ordering of the input schema,
+ * this method always returns columns sorted by field ID.
+ */
+ public static Schema selectInIdOrder(Schema schema, Set fieldIds) {
+ Schema selected = select(schema, fieldIds);
+ List sorted = Lists.newArrayList(selected.columns());
+ sorted.sort(Comparator.comparingInt(Types.NestedField::fieldId));
+ return new Schema(sorted);
+ }
+
public static Types.StructType select(Types.StructType struct, Set fieldIds) {
Preconditions.checkNotNull(struct, "Struct cannot be null");
Preconditions.checkNotNull(fieldIds, "Field ids cannot be null");
@@ -601,6 +615,52 @@ public interface GetID {
int get(int oldId);
}
+ /**
+ * Creates a function that reassigns specified field IDs.
+ *
+ * This is useful for merging schemas where some field IDs in one schema might conflict with
+ * IDs already in use by another schema. The function will reassign the provided IDs to new unused
+ * IDs, while preserving other IDs.
+ *
+ * @param conflictingIds the set of conflicting field IDs that should be reassigned
+ * @param allUsedIds the set of field IDs that are already in use and cannot be reused
+ * @return a function that reassigns conflicting field IDs while preserving others
+ */
+ public static GetID reassignConflictingIds(Set conflictingIds, Set allUsedIds) {
+ return new ReassignConflictingIds(conflictingIds, allUsedIds);
+ }
+
+ private static class ReassignConflictingIds implements GetID {
+ private final Set conflictingIds;
+ private final Set allUsedIds;
+ private final AtomicInteger nextId;
+
+ private ReassignConflictingIds(Set conflictingIds, Set allUsedIds) {
+ this.conflictingIds = conflictingIds;
+ this.allUsedIds = allUsedIds;
+ this.nextId = new AtomicInteger();
+ }
+
+ @Override
+ public int get(int oldId) {
+ if (conflictingIds.contains(oldId)) {
+ return nextAvailableId();
+ } else {
+ return oldId;
+ }
+ }
+
+ private int nextAvailableId() {
+ int candidateId = nextId.incrementAndGet();
+
+ while (allUsedIds.contains(candidateId)) {
+ candidateId = nextId.incrementAndGet();
+ }
+
+ return candidateId;
+ }
+ }
+
public static class SchemaVisitor {
public void beforeField(Types.NestedField field) {}
diff --git a/api/src/main/java/org/apache/iceberg/util/CharSequenceSet.java b/api/src/main/java/org/apache/iceberg/util/CharSequenceSet.java
index cfdac0104c47..c13cbfd0cc28 100644
--- a/api/src/main/java/org/apache/iceberg/util/CharSequenceSet.java
+++ b/api/src/main/java/org/apache/iceberg/util/CharSequenceSet.java
@@ -18,183 +18,53 @@
*/
package org.apache.iceberg.util;
-import java.io.Serializable;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Set;
-import java.util.stream.Collectors;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
-import org.apache.iceberg.relocated.com.google.common.collect.Iterators;
-import org.apache.iceberg.relocated.com.google.common.collect.Sets;
-import org.apache.iceberg.relocated.com.google.common.collect.Streams;
-public class CharSequenceSet implements Set, Serializable {
+public class CharSequenceSet extends WrapperSet {
private static final ThreadLocal WRAPPERS =
ThreadLocal.withInitial(() -> CharSequenceWrapper.wrap(null));
- public static CharSequenceSet of(Iterable charSequences) {
- return new CharSequenceSet(charSequences);
- }
-
- public static CharSequenceSet empty() {
- return new CharSequenceSet(ImmutableList.of());
- }
-
- private final Set wrapperSet;
-
- private CharSequenceSet(Iterable charSequences) {
- this.wrapperSet =
- Sets.newHashSet(Iterables.transform(charSequences, CharSequenceWrapper::wrap));
+ private CharSequenceSet() {
+ // needed for serialization
}
- @Override
- public int size() {
- return wrapperSet.size();
+ private CharSequenceSet(Iterable charSequences) {
+ super(
+ Iterables.transform(
+ charSequences,
+ obj -> {
+ Preconditions.checkNotNull(obj, "Invalid object: null");
+ return CharSequenceWrapper.wrap(obj);
+ }));
}
- @Override
- public boolean isEmpty() {
- return wrapperSet.isEmpty();
+ public static CharSequenceSet of(Iterable charSequences) {
+ return new CharSequenceSet(charSequences);
}
- @Override
- public boolean contains(Object obj) {
- if (obj instanceof CharSequence) {
- CharSequenceWrapper wrapper = WRAPPERS.get();
- boolean result = wrapperSet.contains(wrapper.set((CharSequence) obj));
- wrapper.set(null); // don't hold a reference to the value
- return result;
- }
- return false;
+ public static CharSequenceSet empty() {
+ return new CharSequenceSet();
}
@Override
- public Iterator iterator() {
- return Iterators.transform(wrapperSet.iterator(), CharSequenceWrapper::get);
+ protected Wrapper wrapper() {
+ return WRAPPERS.get();
}
@Override
- public Object[] toArray() {
- return Iterators.toArray(iterator(), CharSequence.class);
+ protected Wrapper wrap(CharSequence file) {
+ return CharSequenceWrapper.wrap(file);
}
@Override
- @SuppressWarnings("unchecked")
- public T[] toArray(T[] destArray) {
- int size = wrapperSet.size();
- if (destArray.length < size) {
- return (T[]) toArray();
- }
-
- Iterator iter = iterator();
- int ind = 0;
- while (iter.hasNext()) {
- destArray[ind] = (T) iter.next();
- ind += 1;
- }
-
- if (destArray.length > size) {
- destArray[size] = null;
- }
-
- return destArray;
+ protected Class elementClass() {
+ return CharSequence.class;
}
@Override
public boolean add(CharSequence charSequence) {
- return wrapperSet.add(CharSequenceWrapper.wrap(charSequence));
- }
-
- @Override
- public boolean remove(Object obj) {
- if (obj instanceof CharSequence) {
- CharSequenceWrapper wrapper = WRAPPERS.get();
- boolean result = wrapperSet.remove(wrapper.set((CharSequence) obj));
- wrapper.set(null); // don't hold a reference to the value
- return result;
- }
- return false;
- }
-
- @Override
- @SuppressWarnings("CollectionUndefinedEquality")
- public boolean containsAll(Collection objects) {
- if (objects != null) {
- return Iterables.all(objects, this::contains);
- }
- return false;
- }
-
- @Override
- public boolean addAll(Collection charSequences) {
- if (charSequences != null) {
- return Iterables.addAll(
- wrapperSet, Iterables.transform(charSequences, CharSequenceWrapper::wrap));
- }
- return false;
- }
-
- @Override
- public boolean retainAll(Collection objects) {
- if (objects != null) {
- Set toRetain =
- objects.stream()
- .filter(CharSequence.class::isInstance)
- .map(CharSequence.class::cast)
- .map(CharSequenceWrapper::wrap)
- .collect(Collectors.toSet());
-
- return Iterables.retainAll(wrapperSet, toRetain);
- }
-
- return false;
- }
-
- @Override
- @SuppressWarnings("CollectionUndefinedEquality")
- public boolean removeAll(Collection objects) {
- if (objects != null) {
- return objects.stream().filter(this::remove).count() != 0;
- }
-
- return false;
- }
-
- @Override
- public void clear() {
- wrapperSet.clear();
- }
-
- @SuppressWarnings("CollectionUndefinedEquality")
- @Override
- public boolean equals(Object other) {
- if (this == other) {
- return true;
- } else if (!(other instanceof Set)) {
- return false;
- }
-
- Set that = (Set) other;
-
- if (size() != that.size()) {
- return false;
- }
-
- try {
- return containsAll(that);
- } catch (ClassCastException | NullPointerException unused) {
- return false;
- }
- }
-
- @Override
- public int hashCode() {
- return wrapperSet.stream().mapToInt(CharSequenceWrapper::hashCode).sum();
- }
-
- @Override
- public String toString() {
- return Streams.stream(iterator()).collect(Collectors.joining("CharSequenceSet({", ", ", "})"));
+ // method is needed to not break API compatibility
+ return super.add(charSequence);
}
}
diff --git a/api/src/main/java/org/apache/iceberg/util/CharSequenceWrapper.java b/api/src/main/java/org/apache/iceberg/util/CharSequenceWrapper.java
index 854264c1ae21..59e8eb712dc6 100644
--- a/api/src/main/java/org/apache/iceberg/util/CharSequenceWrapper.java
+++ b/api/src/main/java/org/apache/iceberg/util/CharSequenceWrapper.java
@@ -18,12 +18,11 @@
*/
package org.apache.iceberg.util;
-import java.io.Serializable;
import org.apache.iceberg.types.Comparators;
import org.apache.iceberg.types.JavaHashes;
/** Wrapper class to adapt CharSequence for use in maps and sets. */
-public class CharSequenceWrapper implements CharSequence, Serializable {
+public class CharSequenceWrapper implements CharSequence, WrapperSet.Wrapper {
public static CharSequenceWrapper wrap(CharSequence seq) {
return new CharSequenceWrapper(seq);
}
@@ -39,6 +38,7 @@ private CharSequenceWrapper(CharSequence wrapped) {
this.wrapped = wrapped;
}
+ @Override
public CharSequenceWrapper set(CharSequence newWrapped) {
this.wrapped = newWrapped;
this.hashCode = 0;
@@ -46,6 +46,7 @@ public CharSequenceWrapper set(CharSequence newWrapped) {
return this;
}
+ @Override
public CharSequence get() {
return wrapped;
}
diff --git a/api/src/test/java/org/apache/iceberg/TestFileContent.java b/api/src/test/java/org/apache/iceberg/TestFileContent.java
new file mode 100644
index 000000000000..bd5e44ed3cf3
--- /dev/null
+++ b/api/src/test/java/org/apache/iceberg/TestFileContent.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.util.stream.IntStream;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.MethodSource;
+
+class TestFileContent {
+
+ @ParameterizedTest
+ @EnumSource(FileContent.class)
+ void fromId(FileContent content) {
+ assertThat(FileContent.fromId(content.id())).isEqualTo(content);
+ }
+
+ static IntStream invalidContentTypeIds() {
+ return IntStream.of(-1, FileContent.values().length);
+ }
+
+ @ParameterizedTest
+ @MethodSource("invalidContentTypeIds")
+ void fromIdInvalid(int id) {
+ assertThatThrownBy(() -> FileContent.fromId(id))
+ .isInstanceOf(ArrayIndexOutOfBoundsException.class)
+ .hasMessageContaining(String.valueOf(id));
+ }
+}
diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java
index b8e16a9ee45e..a1709d2a2e06 100644
--- a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java
+++ b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java
@@ -242,6 +242,22 @@ public void testSettingPartitionTransformsWithCustomTargetNamesThatAlreadyExist(
"Cannot create identity partition sourced from different field in schema: another_ts");
}
+ @Test
+ public void testStalePartitionSourceIdWithReusedColumnName() {
+ int newFieldId = 2;
+ int droppedFieldId = 1;
+ Schema schema =
+ new Schema(NestedField.required(newFieldId, "category", Types.StringType.get()));
+ PartitionSpec spec =
+ PartitionSpec.builderFor(schema)
+ .withSpecId(0)
+ .add(droppedFieldId, 1000, "category", Transforms.alwaysNull())
+ .build();
+ assertThat(spec.fields()).hasSize(1);
+ assertThat(spec.fields().get(0).sourceId()).isEqualTo(droppedFieldId);
+ assertThat(spec.fields().get(0).name()).isEqualTo("category");
+ }
+
@Test
public void testMissingSourceColumn() {
assertThatThrownBy(() -> PartitionSpec.builderFor(SCHEMA).year("missing").build())
diff --git a/api/src/test/java/org/apache/iceberg/TestSchema.java b/api/src/test/java/org/apache/iceberg/TestSchema.java
index 4b164f963d4a..7abc3505d52e 100644
--- a/api/src/test/java/org/apache/iceberg/TestSchema.java
+++ b/api/src/test/java/org/apache/iceberg/TestSchema.java
@@ -21,10 +21,12 @@
import static org.apache.iceberg.Schema.DEFAULT_VALUES_MIN_FORMAT_VERSION;
import static org.apache.iceberg.Schema.MIN_FORMAT_VERSIONS;
import static org.apache.iceberg.TestHelpers.MAX_FORMAT_VERSION;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.List;
+import java.util.Map;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.apache.iceberg.expressions.Literal;
@@ -217,4 +219,97 @@ public void testSupportedWriteDefault(int formatVersion) {
assertThatCode(() -> Schema.checkCompatibility(WRITE_DEFAULT_SCHEMA, formatVersion))
.doesNotThrowAnyException();
}
+
+ @Test
+ public void testIndexFieldsSingleSchema() {
+ Schema schema =
+ new Schema(
+ 1,
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.optional(2, "data", Types.StringType.get()));
+
+ Map fields = Schema.indexFields(ImmutableList.of(schema));
+
+ assertThat(fields).hasSize(2);
+ assertThat(fields.get(1).name()).isEqualTo("id");
+ assertThat(fields.get(2).name()).isEqualTo("data");
+ }
+
+ @Test
+ public void testIndexFieldsHigherSchemaIdTakesPrecedence() {
+ Schema schema1 =
+ new Schema(
+ 1,
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.optional(2, "data", Types.StringType.get()));
+
+ Schema schema2 =
+ new Schema(
+ 2,
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.required(2, "data", Types.IntegerType.get()));
+
+ Map fields = Schema.indexFields(ImmutableList.of(schema2, schema1));
+
+ assertThat(fields).hasSize(2);
+ assertThat(fields.get(2).type()).isEqualTo(Types.IntegerType.get());
+ assertThat(fields.get(2).isOptional()).isFalse();
+ }
+
+ @Test
+ public void testIndexFieldsDuplicateSchemaIds() {
+ Schema schema1 =
+ new Schema(
+ 1,
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.optional(2, "data", Types.StringType.get()));
+
+ Schema schema1Duplicate =
+ new Schema(
+ 1,
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.optional(2, "different", Types.IntegerType.get()));
+
+ Map fields =
+ Schema.indexFields(ImmutableList.of(schema1, schema1Duplicate));
+
+ assertThat(fields).hasSize(2);
+ assertThat(fields.get(2).name()).isEqualTo("data");
+ }
+
+ @Test
+ public void testIndexFieldsNestedSchema() {
+ Schema schema1 =
+ new Schema(
+ 1,
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.optional(
+ 2,
+ "person",
+ Types.StructType.of(
+ Types.NestedField.optional(3, "name", Types.StringType.get()),
+ Types.NestedField.optional(4, "age", Types.IntegerType.get()))));
+
+ Schema schema2 =
+ new Schema(
+ 2,
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.optional(
+ 2,
+ "person",
+ Types.StructType.of(
+ Types.NestedField.optional(3, "name", Types.StringType.get()),
+ Types.NestedField.optional(4, "age", Types.IntegerType.get()),
+ Types.NestedField.optional(5, "email", Types.StringType.get()))));
+
+ Map fields = Schema.indexFields(ImmutableList.of(schema1, schema2));
+
+ assertThat(fields).hasSize(5);
+ assertThat(fields.get(1).name()).isEqualTo("id");
+ assertThat(fields.get(2).name()).isEqualTo("person");
+ assertThat(fields.get(3).name()).isEqualTo("name");
+ assertThat(fields.get(4).name()).isEqualTo("age");
+ assertThat(fields.get(5).name()).isEqualTo("email");
+ assertThat(((Types.StructType) fields.get(2).type()).fields()).hasSize(3);
+ }
}
diff --git a/api/src/test/java/org/apache/iceberg/catalog/TestTableIdentifier.java b/api/src/test/java/org/apache/iceberg/catalog/TestTableIdentifier.java
index ca9569436bab..13781ccaa7f4 100644
--- a/api/src/test/java/org/apache/iceberg/catalog/TestTableIdentifier.java
+++ b/api/src/test/java/org/apache/iceberg/catalog/TestTableIdentifier.java
@@ -22,6 +22,7 @@
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import org.junit.jupiter.api.Test;
+import org.junitpioneer.jupiter.DefaultLocale;
public class TestTableIdentifier {
@@ -52,6 +53,13 @@ public void testToLowerCase() {
.isEqualTo(TableIdentifier.of("Catalog", "dB", "TBL").toLowerCase());
}
+ @Test
+ @DefaultLocale(language = "tr")
+ public void testToLowerCaseIsLocaleIndependent() {
+ assertThat(TableIdentifier.of("information", "db", "tbl"))
+ .isEqualTo(TableIdentifier.of("INFORMATION", "DB", "TBL").toLowerCase());
+ }
+
@Test
public void testInvalidTableName() {
assertThatThrownBy(() -> TableIdentifier.of(Namespace.empty(), ""))
diff --git a/api/src/test/java/org/apache/iceberg/encryption/TestEncryptingFileIO.java b/api/src/test/java/org/apache/iceberg/encryption/TestEncryptingFileIO.java
new file mode 100644
index 000000000000..983203e2db80
--- /dev/null
+++ b/api/src/test/java/org/apache/iceberg/encryption/TestEncryptingFileIO.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.encryption;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
+
+import java.util.Map;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.io.FileInfo;
+import org.apache.iceberg.io.SupportsPrefixOperations;
+import org.junit.jupiter.api.Test;
+
+public class TestEncryptingFileIO {
+
+ @Test
+ public void delegateEncryptingIOWithAndWithoutMixins() {
+ EncryptionManager em = mock(EncryptionManager.class);
+
+ FileIO fileIONoMixins = mock(FileIO.class);
+ assertThat(EncryptingFileIO.combine(fileIONoMixins, em))
+ .isInstanceOf(EncryptingFileIO.class)
+ .extracting(EncryptingFileIO::encryptionManager)
+ .isEqualTo(em);
+
+ FileIO fileIOWithMixins =
+ mock(FileIO.class, withSettings().extraInterfaces(SupportsPrefixOperations.class));
+ assertThat(EncryptingFileIO.combine(fileIOWithMixins, em))
+ .isInstanceOf(EncryptingFileIO.WithSupportsPrefixOperations.class)
+ .extracting(EncryptingFileIO::encryptionManager)
+ .isEqualTo(em);
+ }
+
+ @Test
+ public void prefixOperationsDelegation() {
+ EncryptionManager em = mock(EncryptionManager.class);
+ SupportsPrefixOperations delegate = mock(SupportsPrefixOperations.class);
+
+ EncryptingFileIO.WithSupportsPrefixOperations fileIO =
+ (EncryptingFileIO.WithSupportsPrefixOperations) EncryptingFileIO.combine(delegate, em);
+
+ String prefix = "prefix";
+ Iterable fileInfos = mock(Iterable.class);
+ when(delegate.listPrefix(prefix)).thenReturn(fileInfos);
+ assertThat(fileIO.listPrefix(prefix)).isEqualTo(fileInfos);
+
+ fileIO.deletePrefix(prefix);
+ verify(delegate).deletePrefix(prefix);
+ }
+
+ @Test
+ public void properties() {
+ EncryptionManager em = mock(EncryptionManager.class);
+ FileIO io = mock(FileIO.class);
+ when(io.properties()).thenReturn(Map.of("key", "value"));
+
+ assertThat(EncryptingFileIO.combine(io, em).properties())
+ .containsExactly(Map.entry("key", "value"));
+ }
+}
diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java
index ca08951b1f53..fdf3d9dcd1b0 100644
--- a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java
+++ b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java
@@ -19,6 +19,8 @@
package org.apache.iceberg.expressions;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.nio.ByteBuffer;
import java.time.LocalDate;
@@ -33,6 +35,7 @@
import java.util.stream.IntStream;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.types.Types;
@@ -66,6 +69,24 @@ public class TestExpressionUtil {
private static final Types.StructType FLOAT_TEST =
Types.StructType.of(Types.NestedField.optional(1, "test", Types.FloatType.get()));
+ /** Schema with struct, list, and map columns for {@link #testExtractByIdInclusiveNestedTypes}. */
+ private static final Schema NESTED_EXTRACT_SCHEMA =
+ new Schema(
+ Types.NestedField.required(1, "top_id", Types.LongType.get()),
+ Types.NestedField.optional(
+ 2,
+ "st",
+ Types.StructType.of(
+ Types.NestedField.required(3, "inner_i", Types.IntegerType.get()))),
+ Types.NestedField.optional(
+ 4, "arr", Types.ListType.ofRequired(5, Types.IntegerType.get())),
+ Types.NestedField.optional(
+ 6,
+ "mp",
+ Types.MapType.ofRequired(7, 8, Types.StringType.get(), Types.IntegerType.get())));
+
+ private static final Types.StructType NESTED_EXTRACT_STRUCT = NESTED_EXTRACT_SCHEMA.asStruct();
+
@Test
public void testUnchangedUnaryPredicates() {
for (Expression unary :
@@ -115,6 +136,16 @@ public void testSanitizeLongIn() {
.as("Sanitized string should be abbreviated")
.isEqualTo("test IN ((2-digit-int), (3-digit-int), ... (8 values hidden, 10 in total))");
+ Object[] tooLongStringsList =
+ IntStream.range(0, ExpressionUtil.LONG_IN_PREDICATE_ABBREVIATION_THRESHOLD + 5)
+ .mapToObj(i -> "string_" + i)
+ .toArray();
+
+ assertThat(ExpressionUtil.toSanitizedString(Expressions.in("test", tooLongStringsList)))
+ .as("Sanitized string should be abbreviated")
+ .isEqualTo(
+ "test IN ((hash-14128790), (hash-1056a62b), (hash-22fd6340), (hash-3f9d20e4), (hash-136200f0), (hash-25fc9033), (hash-681d31e2), (hash-6c1796d4), (hash-382d143e), (hash-272f4e5b), ... (5 values hidden, 15 in total))");
+
// The sanitization resulting in an expression tree does not abbreviate
List expectedValues = Lists.newArrayList();
expectedValues.addAll(Collections.nCopies(5, "(2-digit-int)"));
@@ -812,6 +843,146 @@ public void testSanitizeStringFallback() {
}
}
+ @Test
+ public void testExtractByIdInclusive() {
+ Expression alwaysTrue = Expressions.alwaysTrue();
+ Expression idEq = Expressions.equal("id", 5L);
+ Expression valEq = Expressions.equal("val", 5);
+
+ assertThat(
+ ExpressionUtil.equivalent(
+ alwaysTrue,
+ ExpressionUtil.extractByIdInclusive(
+ Expressions.and(idEq, valEq), SCHEMA, true, new int[0]),
+ STRUCT,
+ true))
+ .isTrue();
+
+ assertThat(
+ ExpressionUtil.equivalent(
+ alwaysTrue,
+ ExpressionUtil.extractByIdInclusive(
+ Expressions.and(idEq, valEq), SCHEMA, true, (int[]) null),
+ STRUCT,
+ true))
+ .isTrue();
+
+ assertThat(
+ ExpressionUtil.equivalent(
+ idEq, ExpressionUtil.extractByIdInclusive(idEq, SCHEMA, true, 1), STRUCT, true))
+ .isTrue();
+
+ assertThat(
+ ExpressionUtil.equivalent(
+ alwaysTrue,
+ ExpressionUtil.extractByIdInclusive(valEq, SCHEMA, true, 1),
+ STRUCT,
+ true))
+ .isTrue();
+
+ assertThat(
+ ExpressionUtil.equivalent(
+ idEq,
+ ExpressionUtil.extractByIdInclusive(Expressions.and(idEq, valEq), SCHEMA, true, 1),
+ STRUCT,
+ true))
+ .isTrue();
+
+ Expression orBothId = Expressions.or(Expressions.equal("id", 1L), Expressions.equal("id", 2L));
+ assertThat(
+ ExpressionUtil.equivalent(
+ orBothId,
+ ExpressionUtil.extractByIdInclusive(orBothId, SCHEMA, true, 1),
+ STRUCT,
+ true))
+ .isTrue();
+ }
+
+ @Test
+ public void testExtractByIdInclusiveNestedTypes() {
+ Expression alwaysTrue = Expressions.alwaysTrue();
+ Expression structPred = Expressions.equal("st.inner_i", 1);
+ Expression listPred = Expressions.equal("arr.element", 42);
+ Expression mapKeyPred = Expressions.equal("mp.key", "k");
+ Expression mapValuePred = Expressions.equal("mp.value", 7);
+ Expression topPred = Expressions.equal("top_id", 9L);
+
+ assertThat(
+ ExpressionUtil.equivalent(
+ structPred,
+ ExpressionUtil.extractByIdInclusive(structPred, NESTED_EXTRACT_SCHEMA, true, 3),
+ NESTED_EXTRACT_STRUCT,
+ true))
+ .isTrue();
+ assertThat(
+ ExpressionUtil.equivalent(
+ alwaysTrue,
+ ExpressionUtil.extractByIdInclusive(structPred, NESTED_EXTRACT_SCHEMA, true, 1),
+ NESTED_EXTRACT_STRUCT,
+ true))
+ .isTrue();
+
+ assertThat(
+ ExpressionUtil.equivalent(
+ listPred,
+ ExpressionUtil.extractByIdInclusive(listPred, NESTED_EXTRACT_SCHEMA, true, 5),
+ NESTED_EXTRACT_STRUCT,
+ true))
+ .isTrue();
+ assertThat(
+ ExpressionUtil.equivalent(
+ alwaysTrue,
+ ExpressionUtil.extractByIdInclusive(listPred, NESTED_EXTRACT_SCHEMA, true, 1),
+ NESTED_EXTRACT_STRUCT,
+ true))
+ .isTrue();
+
+ assertThat(
+ ExpressionUtil.equivalent(
+ mapKeyPred,
+ ExpressionUtil.extractByIdInclusive(mapKeyPred, NESTED_EXTRACT_SCHEMA, true, 7),
+ NESTED_EXTRACT_STRUCT,
+ true))
+ .isTrue();
+ assertThat(
+ ExpressionUtil.equivalent(
+ mapValuePred,
+ ExpressionUtil.extractByIdInclusive(mapValuePred, NESTED_EXTRACT_SCHEMA, true, 8),
+ NESTED_EXTRACT_STRUCT,
+ true))
+ .isTrue();
+ assertThat(
+ ExpressionUtil.equivalent(
+ alwaysTrue,
+ ExpressionUtil.extractByIdInclusive(mapKeyPred, NESTED_EXTRACT_SCHEMA, true, 8),
+ NESTED_EXTRACT_STRUCT,
+ true))
+ .isTrue();
+
+ Expression mixed = Expressions.and(structPred, Expressions.and(listPred, topPred));
+ assertThat(
+ ExpressionUtil.equivalent(
+ structPred,
+ ExpressionUtil.extractByIdInclusive(mixed, NESTED_EXTRACT_SCHEMA, true, 3),
+ NESTED_EXTRACT_STRUCT,
+ true))
+ .isTrue();
+ assertThat(
+ ExpressionUtil.equivalent(
+ listPred,
+ ExpressionUtil.extractByIdInclusive(mixed, NESTED_EXTRACT_SCHEMA, true, 5),
+ NESTED_EXTRACT_STRUCT,
+ true))
+ .isTrue();
+ assertThat(
+ ExpressionUtil.equivalent(
+ topPred,
+ ExpressionUtil.extractByIdInclusive(mixed, NESTED_EXTRACT_SCHEMA, true, 1),
+ NESTED_EXTRACT_STRUCT,
+ true))
+ .isTrue();
+ }
+
@Test
public void testIdenticalExpressionIsEquivalent() {
Expression[] exprs =
@@ -1062,6 +1233,22 @@ public void testSelectsPartitions() {
.isFalse();
}
+ @Test
+ public void testSelectsPartitionsWithUnpartitionedTable() {
+ Table table = mock(Table.class);
+ Map specs =
+ ImmutableMap.of(
+ 0,
+ PartitionSpec.unpartitioned(),
+ 1,
+ PartitionSpec.builderFor(SCHEMA).identity("val").build());
+ when(table.specs()).thenReturn(specs);
+
+ assertThat(ExpressionUtil.selectsPartitions(Expressions.lessThan("id", 1L), table, true))
+ .as("Should return false for unpartitioned table (no partition boundaries to select)")
+ .isFalse();
+ }
+
@Test
public void testSanitizeVariantArray() {
Expression bound =
diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveManifestEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveManifestEvaluator.java
index 068c862e2bda..78e6064eb427 100644
--- a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveManifestEvaluator.java
+++ b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveManifestEvaluator.java
@@ -66,7 +66,10 @@ public class TestInclusiveManifestEvaluator {
optional(12, "no_nan_or_null", Types.DoubleType.get()),
optional(13, "all_nulls_missing_nan_float", Types.FloatType.get()),
optional(14, "all_same_value_or_null", Types.StringType.get()),
- optional(15, "no_nulls_same_value_a", Types.StringType.get()));
+ optional(15, "no_nulls_same_value_a", Types.StringType.get()),
+ optional(16, "single_value_with_nan", Types.FloatType.get()),
+ optional(17, "single_value_nan_unknown", Types.FloatType.get()),
+ optional(18, "single_value_no_nan", Types.FloatType.get()));
private static final PartitionSpec SPEC =
PartitionSpec.builderFor(SCHEMA)
@@ -84,6 +87,9 @@ public class TestInclusiveManifestEvaluator {
.identity("all_nulls_missing_nan_float")
.identity("all_same_value_or_null")
.identity("no_nulls_same_value_a")
+ .identity("single_value_with_nan")
+ .identity("single_value_nan_unknown")
+ .identity("single_value_no_nan")
.build();
private static final int INT_MIN_VALUE = 30;
@@ -128,7 +134,21 @@ public class TestInclusiveManifestEvaluator {
toByteBuffer(Types.FloatType.get(), 20F)),
new TestHelpers.TestFieldSummary(true, null, null),
new TestHelpers.TestFieldSummary(true, STRING_MIN, STRING_MIN),
- new TestHelpers.TestFieldSummary(false, STRING_MIN, STRING_MIN)),
+ new TestHelpers.TestFieldSummary(false, STRING_MIN, STRING_MIN),
+ new TestHelpers.TestFieldSummary(
+ false,
+ true,
+ toByteBuffer(Types.FloatType.get(), 5.0F),
+ toByteBuffer(Types.FloatType.get(), 5.0F)),
+ new TestHelpers.TestFieldSummary(
+ false,
+ toByteBuffer(Types.FloatType.get(), 5.0F),
+ toByteBuffer(Types.FloatType.get(), 5.0F)),
+ new TestHelpers.TestFieldSummary(
+ false,
+ false,
+ toByteBuffer(Types.FloatType.get(), 5.0F),
+ toByteBuffer(Types.FloatType.get(), 5.0F))),
null);
@Test
@@ -353,7 +373,7 @@ public void testIntegerLt() {
shouldRead =
ManifestEvaluator.forRowFilter(lessThan("id", INT_MAX_VALUE), SPEC, true).eval(FILE);
- assertThat(shouldRead).as("Should read: may possible ids").isTrue();
+ assertThat(shouldRead).as("Should read: many possible ids").isTrue();
}
@Test
@@ -395,7 +415,7 @@ public void testIntegerGt() {
shouldRead =
ManifestEvaluator.forRowFilter(greaterThan("id", INT_MAX_VALUE - 4), SPEC, true).eval(FILE);
- assertThat(shouldRead).as("Should read: may possible ids").isTrue();
+ assertThat(shouldRead).as("Should read: many possible ids").isTrue();
}
@Test
@@ -418,7 +438,7 @@ public void testIntegerGtEq() {
shouldRead =
ManifestEvaluator.forRowFilter(greaterThanOrEqual("id", INT_MAX_VALUE - 4), SPEC, true)
.eval(FILE);
- assertThat(shouldRead).as("Should read: may possible ids").isTrue();
+ assertThat(shouldRead).as("Should read: many possible ids").isTrue();
}
@Test
@@ -753,4 +773,84 @@ public void testIntegerNotIn() {
ManifestEvaluator.forRowFilter(notIn("no_nulls", "abc", "def"), SPEC, true).eval(FILE);
assertThat(shouldRead).as("Should read: notIn on no nulls column").isTrue();
}
+
+ @Test
+ public void testNotEqWithSingleValue() {
+ boolean shouldRead =
+ ManifestEvaluator.forRowFilter(notEqual("no_nulls", "a"), SPEC, true).eval(FILE);
+ assertThat(shouldRead).as("Should read: manifest has range of values").isTrue();
+ shouldRead =
+ ManifestEvaluator.forRowFilter(notEqual("no_nulls_same_value_a", "a"), SPEC, true)
+ .eval(FILE);
+ assertThat(shouldRead)
+ .as("Should not read: manifest contains single value equal to literal")
+ .isFalse();
+ shouldRead =
+ ManifestEvaluator.forRowFilter(notEqual("no_nulls_same_value_a", "b"), SPEC, true)
+ .eval(FILE);
+ assertThat(shouldRead)
+ .as("Should read: manifest contains single value not equal to literal")
+ .isTrue();
+ shouldRead =
+ ManifestEvaluator.forRowFilter(notEqual("all_same_value_or_null", "a"), SPEC, true)
+ .eval(FILE);
+ assertThat(shouldRead).as("Should read: manifest has nulls which match != predicate").isTrue();
+ shouldRead =
+ ManifestEvaluator.forRowFilter(notEqual("single_value_with_nan", 5.0F), SPEC, true)
+ .eval(FILE);
+ assertThat(shouldRead)
+ .as("Should read: manifest has NaN values which match != predicate")
+ .isTrue();
+ shouldRead =
+ ManifestEvaluator.forRowFilter(notEqual("single_value_nan_unknown", 5.0F), SPEC, true)
+ .eval(FILE);
+ assertThat(shouldRead).as("Should read: manifest has unknown NaN info").isTrue();
+ shouldRead =
+ ManifestEvaluator.forRowFilter(notEqual("single_value_no_nan", 5.0F), SPEC, true)
+ .eval(FILE);
+ assertThat(shouldRead)
+ .as("Should not read: manifest contains single float value with no NaNs")
+ .isFalse();
+ }
+
+ @Test
+ public void testNotInWithSingleValue() {
+ boolean shouldRead =
+ ManifestEvaluator.forRowFilter(notIn("no_nulls", "a", "b"), SPEC, true).eval(FILE);
+ assertThat(shouldRead).as("Should read: manifest has range of values").isTrue();
+ shouldRead =
+ ManifestEvaluator.forRowFilter(notIn("no_nulls_same_value_a", "a", "b"), SPEC, true)
+ .eval(FILE);
+ assertThat(shouldRead)
+ .as("Should not read: manifest contains single value in exclusion list")
+ .isFalse();
+ shouldRead =
+ ManifestEvaluator.forRowFilter(notIn("no_nulls_same_value_a", "b", "c"), SPEC, true)
+ .eval(FILE);
+ assertThat(shouldRead)
+ .as("Should read: manifest contains single value not in exclusion list")
+ .isTrue();
+ shouldRead =
+ ManifestEvaluator.forRowFilter(notIn("all_same_value_or_null", "a", "b"), SPEC, true)
+ .eval(FILE);
+ assertThat(shouldRead)
+ .as("Should read: manifest has nulls which match NOT IN predicate")
+ .isTrue();
+ shouldRead =
+ ManifestEvaluator.forRowFilter(notIn("single_value_with_nan", 5.0F, 10.0F), SPEC, true)
+ .eval(FILE);
+ assertThat(shouldRead)
+ .as("Should read: manifest has NaN values which match NOT IN predicate")
+ .isTrue();
+ shouldRead =
+ ManifestEvaluator.forRowFilter(notIn("single_value_nan_unknown", 5.0F, 10.0F), SPEC, true)
+ .eval(FILE);
+ assertThat(shouldRead).as("Should read: manifest has unknown NaN info").isTrue();
+ shouldRead =
+ ManifestEvaluator.forRowFilter(notIn("single_value_no_nan", 5.0F, 10.0F), SPEC, true)
+ .eval(FILE);
+ assertThat(shouldRead)
+ .as("Should not read: manifest contains single float value with no NaNs")
+ .isFalse();
+ }
}
diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java
index 2f4fbf395739..5f0ca2659fbf 100644
--- a/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java
+++ b/api/src/test/java/org/apache/iceberg/expressions/TestInclusiveMetricsEvaluator.java
@@ -474,7 +474,7 @@ public void testIntegerLt() {
assertThat(shouldRead).as("Should read: one possible id").isTrue();
shouldRead = new InclusiveMetricsEvaluator(SCHEMA, lessThan("id", INT_MAX_VALUE)).eval(FILE);
- assertThat(shouldRead).as("Should read: may possible ids").isTrue();
+ assertThat(shouldRead).as("Should read: many possible ids").isTrue();
}
@Test
@@ -513,7 +513,7 @@ public void testIntegerGt() {
shouldRead =
new InclusiveMetricsEvaluator(SCHEMA, greaterThan("id", INT_MAX_VALUE - 4)).eval(FILE);
- assertThat(shouldRead).as("Should read: may possible ids").isTrue();
+ assertThat(shouldRead).as("Should read: many possible ids").isTrue();
}
@Test
@@ -535,7 +535,7 @@ public void testIntegerGtEq() {
shouldRead =
new InclusiveMetricsEvaluator(SCHEMA, greaterThanOrEqual("id", INT_MAX_VALUE - 4))
.eval(FILE);
- assertThat(shouldRead).as("Should read: may possible ids").isTrue();
+ assertThat(shouldRead).as("Should read: many possible ids").isTrue();
}
@Test
@@ -970,4 +970,172 @@ public void testNotNullInNestedStruct() {
.as("Should not read: optional_address.optional_street2 is optional")
.isFalse();
}
+
+ @Test
+ public void testNotEqWithSingleValue() {
+ DataFile rangeOfValues =
+ new TestDataFile(
+ "range_of_values.avro",
+ Row.of(),
+ 10,
+ ImmutableMap.of(3, 10L),
+ ImmutableMap.of(3, 0L),
+ ImmutableMap.of(3, 0L),
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")),
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz")));
+
+ boolean shouldRead =
+ new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "aaa")).eval(rangeOfValues);
+ assertThat(shouldRead)
+ .as("Should read: file has range of values, cannot exclude based on literal")
+ .isTrue();
+
+ DataFile singleValueFile =
+ new TestDataFile(
+ "single_value.avro",
+ Row.of(),
+ 10,
+ ImmutableMap.of(3, 10L),
+ ImmutableMap.of(3, 0L),
+ ImmutableMap.of(3, 0L),
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")),
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")));
+
+ shouldRead =
+ new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "abc")).eval(singleValueFile);
+ assertThat(shouldRead)
+ .as("Should not read: file contains single value equal to literal")
+ .isFalse();
+
+ shouldRead =
+ new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "def")).eval(singleValueFile);
+ assertThat(shouldRead)
+ .as("Should read: file contains single value not equal to literal")
+ .isTrue();
+
+ DataFile singleValueWithNulls =
+ new TestDataFile(
+ "single_value_nulls.avro",
+ Row.of(),
+ 10,
+ ImmutableMap.of(3, 10L),
+ ImmutableMap.of(3, 2L),
+ ImmutableMap.of(3, 0L),
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")),
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")));
+
+ shouldRead =
+ new InclusiveMetricsEvaluator(SCHEMA, notEqual("required", "abc"))
+ .eval(singleValueWithNulls);
+ assertThat(shouldRead).as("Should read: file has nulls which match != predicate").isTrue();
+
+ DataFile singleValueWithNaN =
+ new TestDataFile(
+ "single_value_nan.avro",
+ Row.of(),
+ 10,
+ ImmutableMap.of(9, 10L),
+ ImmutableMap.of(9, 0L),
+ ImmutableMap.of(9, 2L),
+ ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)),
+ ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)));
+
+ shouldRead =
+ new InclusiveMetricsEvaluator(SCHEMA, notEqual("no_nans", 5.0F)).eval(singleValueWithNaN);
+ assertThat(shouldRead).as("Should read: file has NaN values which match != predicate").isTrue();
+
+ DataFile singleValueNaNBounds =
+ new TestDataFile(
+ "single_value_nan_bounds.avro",
+ Row.of(),
+ 10,
+ ImmutableMap.of(9, 10L),
+ ImmutableMap.of(9, 0L),
+ ImmutableMap.of(9, 0L),
+ ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN)),
+ ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), Float.NaN)));
+
+ shouldRead =
+ new InclusiveMetricsEvaluator(SCHEMA, notEqual("no_nans", 5.0F)).eval(singleValueNaNBounds);
+ assertThat(shouldRead).as("Should read: bounds are NaN").isTrue();
+ }
+
+ @Test
+ public void testNotInWithSingleValue() {
+ DataFile rangeOfValues =
+ new TestDataFile(
+ "range_of_values.avro",
+ Row.of(),
+ 10,
+ ImmutableMap.of(3, 10L),
+ ImmutableMap.of(3, 0L),
+ ImmutableMap.of(3, 0L),
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "aaa")),
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "zzz")));
+
+ boolean shouldRead =
+ new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "aaa", "bbb")).eval(rangeOfValues);
+ assertThat(shouldRead)
+ .as("Should read: file has range of values, cannot exclude based on literal")
+ .isTrue();
+
+ DataFile singleValueFile =
+ new TestDataFile(
+ "single_value.avro",
+ Row.of(),
+ 10,
+ ImmutableMap.of(3, 10L),
+ ImmutableMap.of(3, 0L),
+ ImmutableMap.of(3, 0L),
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")),
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")));
+
+ shouldRead =
+ new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "abc", "def"))
+ .eval(singleValueFile);
+ assertThat(shouldRead)
+ .as("Should not read: file contains single value in exclusion list")
+ .isFalse();
+
+ shouldRead =
+ new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "def", "ghi"))
+ .eval(singleValueFile);
+ assertThat(shouldRead)
+ .as("Should read: file contains single value not in exclusion list")
+ .isTrue();
+
+ DataFile singleValueWithNulls =
+ new TestDataFile(
+ "single_value_nulls.avro",
+ Row.of(),
+ 10,
+ ImmutableMap.of(3, 10L),
+ ImmutableMap.of(3, 2L),
+ ImmutableMap.of(3, 0L),
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")),
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")));
+
+ shouldRead =
+ new InclusiveMetricsEvaluator(SCHEMA, notIn("required", "abc", "def"))
+ .eval(singleValueWithNulls);
+ assertThat(shouldRead).as("Should read: file has nulls which match NOT IN predicate").isTrue();
+
+ DataFile singleValueWithNaN =
+ new TestDataFile(
+ "single_value_nan.avro",
+ Row.of(),
+ 10,
+ ImmutableMap.of(9, 10L),
+ ImmutableMap.of(9, 0L),
+ ImmutableMap.of(9, 2L),
+ ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)),
+ ImmutableMap.of(9, toByteBuffer(Types.FloatType.get(), 5.0F)));
+
+ shouldRead =
+ new InclusiveMetricsEvaluator(SCHEMA, notIn("no_nans", 5.0F, 10.0F))
+ .eval(singleValueWithNaN);
+ assertThat(shouldRead)
+ .as("Should read: file has NaN values which match NOT IN predicate")
+ .isTrue();
+ }
}
diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestMiscLiteralConversions.java b/api/src/test/java/org/apache/iceberg/expressions/TestMiscLiteralConversions.java
index e2611ddb281f..53aec44ac7ce 100644
--- a/api/src/test/java/org/apache/iceberg/expressions/TestMiscLiteralConversions.java
+++ b/api/src/test/java/org/apache/iceberg/expressions/TestMiscLiteralConversions.java
@@ -107,6 +107,62 @@ public void testFixedToBinary() {
.isEqualTo(lit.value().duplicate());
}
+ @Test
+ public void testStringToFixed() {
+ // Test valid hex string to fixed conversion
+ Literal hexString = Literal.of("000102");
+ Literal fixedLit = hexString.to(Types.FixedType.ofLength(3));
+ assertThat(fixedLit).as("Should convert valid hex string to fixed").isNotNull();
+ assertThat(fixedLit.value().array())
+ .as("Should decode hex string correctly")
+ .isEqualTo(new byte[] {0, 1, 2});
+
+ // Test lowercase hex string
+ Literal lowercaseHex = Literal.of("0a0b0c");
+ Literal lowercaseFixed = lowercaseHex.to(Types.FixedType.ofLength(3));
+ assertThat(lowercaseFixed).as("Should convert lowercase hex string to fixed").isNotNull();
+ assertThat(lowercaseFixed.value().array())
+ .as("Should decode lowercase hex string correctly")
+ .isEqualTo(new byte[] {10, 11, 12});
+
+ // Test wrong length returns null
+ Literal wrongLength = Literal.of("0001");
+ assertThat(wrongLength.to(Types.FixedType.ofLength(3)))
+ .as("Should return null for wrong length")
+ .isNull();
+
+ // Test invalid hex string returns null
+ Literal invalidHex = Literal.of("GGHHII");
+ assertThat(invalidHex.to(Types.FixedType.ofLength(3)))
+ .as("Should return null for invalid hex string")
+ .isNull();
+ }
+
+ @Test
+ public void testStringToBinary() {
+ // Test valid hex string to binary conversion
+ Literal hexString = Literal.of("000102");
+ Literal binaryLit = hexString.to(Types.BinaryType.get());
+ assertThat(binaryLit).as("Should convert valid hex string to binary").isNotNull();
+ assertThat(binaryLit.value().array())
+ .as("Should decode hex string correctly")
+ .isEqualTo(new byte[] {0, 1, 2});
+
+ // Test lowercase hex string
+ Literal lowercaseHex = Literal.of("0a0b0c");
+ Literal lowercaseBinary = lowercaseHex.to(Types.BinaryType.get());
+ assertThat(lowercaseBinary).as("Should convert lowercase hex string to binary").isNotNull();
+ assertThat(lowercaseBinary.value().array())
+ .as("Should decode lowercase hex string correctly")
+ .isEqualTo(new byte[] {10, 11, 12});
+
+ // Test invalid hex string returns null
+ Literal invalidHex = Literal.of("GGHHII");
+ assertThat(invalidHex.to(Types.BinaryType.get()))
+ .as("Should return null for invalid hex string")
+ .isNull();
+ }
+
@Test
public void testInvalidBooleanConversions() {
testInvalidConversions(
diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestStrictMetricsEvaluator.java b/api/src/test/java/org/apache/iceberg/expressions/TestStrictMetricsEvaluator.java
index f34cd730df77..b55f4efb1726 100644
--- a/api/src/test/java/org/apache/iceberg/expressions/TestStrictMetricsEvaluator.java
+++ b/api/src/test/java/org/apache/iceberg/expressions/TestStrictMetricsEvaluator.java
@@ -32,7 +32,9 @@
import static org.apache.iceberg.expressions.Expressions.notIn;
import static org.apache.iceberg.expressions.Expressions.notNaN;
import static org.apache.iceberg.expressions.Expressions.notNull;
+import static org.apache.iceberg.expressions.Expressions.notStartsWith;
import static org.apache.iceberg.expressions.Expressions.or;
+import static org.apache.iceberg.expressions.Expressions.startsWith;
import static org.apache.iceberg.types.Conversions.toByteBuffer;
import static org.apache.iceberg.types.Types.NestedField.optional;
import static org.apache.iceberg.types.Types.NestedField.required;
@@ -72,8 +74,8 @@ public class TestStrictMetricsEvaluator {
"struct",
Types.StructType.of(
Types.NestedField.optional(16, "nested_col_no_stats", Types.IntegerType.get()),
- Types.NestedField.optional(
- 17, "nested_col_with_stats", Types.IntegerType.get()))));
+ Types.NestedField.optional(17, "nested_col_with_stats", Types.IntegerType.get()),
+ Types.NestedField.optional(18, "nested_string_col", Types.StringType.get()))));
private static final int INT_MIN_VALUE = 30;
private static final int INT_MAX_VALUE = 79;
@@ -172,6 +174,40 @@ public class TestStrictMetricsEvaluator {
// upper bounds
ImmutableMap.of(5, toByteBuffer(StringType.get(), "bbb")));
+ // String-focused file: required column 3 has no nulls and string bounds ["abc", "abd"]
+ private static final DataFile STRING_FILE =
+ new TestDataFile(
+ "string_file.avro",
+ Row.of(),
+ 50,
+ // any value counts, including nulls
+ ImmutableMap.of(3, 50L),
+ // null value counts
+ ImmutableMap.of(),
+ // nan value counts
+ null,
+ // lower bounds
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "abc")),
+ // upper bounds
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "abd")));
+
+ // String file with wider range: required column 3 has no nulls and bounds ["aa", "dC"]
+ private static final DataFile STRING_FILE_2 =
+ new TestDataFile(
+ "string_file_2.avro",
+ Row.of(),
+ 50,
+ // any value counts, including nulls
+ ImmutableMap.of(3, 50L),
+ // null value counts
+ ImmutableMap.of(),
+ // nan value counts
+ null,
+ // lower bounds
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "aa")),
+ // upper bounds
+ ImmutableMap.of(3, toByteBuffer(StringType.get(), "dC")));
+
@Test
public void testAllNulls() {
boolean shouldRead = new StrictMetricsEvaluator(SCHEMA, notNull("all_nulls")).eval(FILE);
@@ -684,4 +720,205 @@ SCHEMA, lessThanOrEqual("struct.nested_col_with_stats", INT_MAX_VALUE))
new StrictMetricsEvaluator(SCHEMA, notNull("struct.nested_col_with_stats")).eval(FILE);
assertThat(shouldRead).as("notNull nested column should not match").isFalse();
}
+
+ @Test
+ public void testNotStartsWithAllNulls() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("all_nulls", "a")).eval(FILE);
+ assertThat(shouldRead).as("Should match: all null values satisfy notStartsWith").isTrue();
+ }
+
+ @Test
+ public void testNotStartsWithBoundsAbovePrefix() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("required", "aaa")).eval(STRING_FILE);
+ assertThat(shouldRead).as("Should match: all values are above the prefix range").isTrue();
+ }
+
+ @Test
+ public void testNotStartsWithBoundsBelowPrefix() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("required", "zzz")).eval(STRING_FILE);
+ assertThat(shouldRead).as("Should match: all values are below the prefix range").isTrue();
+ }
+
+ @Test
+ public void testNotStartsWithBoundsOverlapPrefix() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("required", "ab")).eval(STRING_FILE);
+ assertThat(shouldRead).as("Should not match: bounds overlap the prefix range").isFalse();
+
+ shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("required", "abc")).eval(STRING_FILE);
+ assertThat(shouldRead).as("Should not match: lower bound starts with the prefix").isFalse();
+ }
+
+ @Test
+ public void testNotStartsWithWiderRange() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("required", "e")).eval(STRING_FILE_2);
+ assertThat(shouldRead).as("Should match: all values are below the prefix").isTrue();
+
+ shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("required", "a")).eval(STRING_FILE_2);
+ assertThat(shouldRead).as("Should not match: lower bound starts with the prefix").isFalse();
+
+ shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("required", "c")).eval(STRING_FILE_2);
+ assertThat(shouldRead).as("Should not match: prefix is within the bounds range").isFalse();
+ }
+
+ @Test
+ public void testNotStartsWithNoStats() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("required", "a")).eval(FILE);
+ assertThat(shouldRead).as("Should not match: no bounds available for column").isFalse();
+ }
+
+ @Test
+ void testStartsWithBothBoundsMatchPrefix() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, startsWith("required", "ab")).eval(STRING_FILE);
+ assertThat(shouldRead).as("Should match: both bounds start with the prefix").isTrue();
+ }
+
+ @Test
+ void testStartsWithSingleCharPrefixBothBoundsMatch() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, startsWith("required", "a")).eval(STRING_FILE);
+ assertThat(shouldRead)
+ .as("Should match: both bounds start with the single char prefix")
+ .isTrue();
+ }
+
+ @Test
+ void testStartsWithOnlyLowerBoundMatchesPrefix() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, startsWith("required", "abc")).eval(STRING_FILE);
+ assertThat(shouldRead)
+ .as("Should not match: upper bound does not start with the prefix")
+ .isFalse();
+ }
+
+ @Test
+ void testStartsWithBoundsDoNotMatchPrefix() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, startsWith("required", "zzz")).eval(STRING_FILE);
+ assertThat(shouldRead).as("Should not match: no bounds start with the prefix").isFalse();
+ }
+
+ @Test
+ void testStartsWithWiderRange() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, startsWith("required", "a")).eval(STRING_FILE_2);
+ assertThat(shouldRead)
+ .as("Should not match: upper bound does not start with the prefix")
+ .isFalse();
+
+ shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, startsWith("required", "e")).eval(STRING_FILE_2);
+ assertThat(shouldRead).as("Should not match: no bounds start with the prefix").isFalse();
+ }
+
+ @Test
+ void testStartsWithNoStats() {
+ boolean shouldRead = new StrictMetricsEvaluator(SCHEMA, startsWith("required", "a")).eval(FILE);
+ assertThat(shouldRead).as("Should not match: no bounds available for column").isFalse();
+ }
+
+ @Test
+ public void testNotStartsWithSomeNullsBoundsOutsidePrefix() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("some_nulls", "zzz")).eval(FILE_2);
+ assertThat(shouldRead).as("Should match: all values are below the prefix").isTrue();
+
+ shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("some_nulls", "aaa")).eval(FILE_2);
+ assertThat(shouldRead).as("Should match: all values are above the prefix").isTrue();
+ }
+
+ @Test
+ public void testNotStartsWithPrefixLongerThanBounds() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("required", "aaaaaaa")).eval(STRING_FILE);
+ assertThat(shouldRead).as("Should match: all values are above the long prefix").isTrue();
+
+ shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("required", "zzzzzzz")).eval(STRING_FILE);
+ assertThat(shouldRead).as("Should match: all values are below the long prefix").isTrue();
+
+ shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("required", "abcdef")).eval(STRING_FILE);
+ assertThat(shouldRead).as("Should not match: prefix overlaps with bound range").isFalse();
+ }
+
+ @Test
+ void testNotStartsWithEmptyPrefix() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("required", "")).eval(STRING_FILE);
+ assertThat(shouldRead).as("Should not match: all strings start with empty prefix").isFalse();
+ }
+
+ @Test
+ void testNotStartsWithExactBoundMatch() {
+ // FILE_3 has column 5 (some_nulls) with exact bounds ["bbb", "bbb"]
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("some_nulls", "bbb")).eval(FILE_3);
+ assertThat(shouldRead).as("Should not match: bounds exactly equal the prefix").isFalse();
+
+ shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("some_nulls", "zzz")).eval(FILE_3);
+ assertThat(shouldRead).as("Should match: all values are below the prefix").isTrue();
+
+ shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("some_nulls", "aaa")).eval(FILE_3);
+ assertThat(shouldRead).as("Should match: all values are above the prefix").isTrue();
+ }
+
+ @Test
+ public void testNotStartsWithNestedColumn() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, notStartsWith("struct.nested_string_col", "a"))
+ .eval(FILE);
+ assertThat(shouldRead).as("notStartsWith nested column should not match").isFalse();
+ }
+
+ @Test
+ void testStartsWithAllNulls() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, startsWith("all_nulls", "a")).eval(FILE);
+ assertThat(shouldRead)
+ .as("Should not match: all null values do not satisfy startsWith")
+ .isFalse();
+ }
+
+ @Test
+ void testStartsWithSomeNulls() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, startsWith("some_nulls", "b")).eval(FILE_2);
+ assertThat(shouldRead)
+ .as("Should not match: some nulls means not all rows can satisfy startsWith")
+ .isFalse();
+ }
+
+ @Test
+ void testStartsWithPrefixLongerThanBounds() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, startsWith("required", "abcdef")).eval(STRING_FILE);
+ assertThat(shouldRead).as("Should not match: prefix is longer than the bounds").isFalse();
+ }
+
+ @Test
+ void testStartsWithEmptyPrefix() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, startsWith("required", "")).eval(STRING_FILE);
+ assertThat(shouldRead).as("Should match: all strings start with empty prefix").isTrue();
+ }
+
+ @Test
+ void testStartsWithNestedColumn() {
+ boolean shouldRead =
+ new StrictMetricsEvaluator(SCHEMA, startsWith("struct.nested_string_col", "a")).eval(FILE);
+ assertThat(shouldRead).as("Should not match: nested column is not supported").isFalse();
+ }
}
diff --git a/api/src/test/java/org/apache/iceberg/io/TestFileRange.java b/api/src/test/java/org/apache/iceberg/io/TestFileRange.java
new file mode 100644
index 000000000000..dc4ede9ec3b4
--- /dev/null
+++ b/api/src/test/java/org/apache/iceberg/io/TestFileRange.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.io;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.io.EOFException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.CompletableFuture;
+import org.junit.jupiter.api.Test;
+
+public class TestFileRange {
+
+ @Test
+ public void validRange() throws EOFException {
+ CompletableFuture future = new CompletableFuture<>();
+ FileRange range = new FileRange(future, 10L, 100);
+ assertThat(range.offset()).isEqualTo(10L);
+ assertThat(range.length()).isEqualTo(100);
+ assertThat(range.byteBuffer()).isSameAs(future);
+ }
+
+ @Test
+ public void negativeLength() {
+ CompletableFuture future = new CompletableFuture<>();
+ assertThatThrownBy(() -> new FileRange(future, 0L, -1))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid length: -1 in range (must be >= 0)");
+ }
+
+ @Test
+ public void negativeOffset() {
+ CompletableFuture future = new CompletableFuture<>();
+ assertThatThrownBy(() -> new FileRange(future, -1L, 0))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Invalid offset: -1 in range (must be >= 0)");
+ }
+
+ @Test
+ public void nullByteBuffer() {
+ assertThatThrownBy(() -> new FileRange(null, 0L, 0))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("byteBuffer can't be null");
+ }
+}
diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java
index d4742f518754..dd8afebab84d 100644
--- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java
+++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java
@@ -317,6 +317,31 @@ public void testSelect() {
assertThat(actualDepthTwo.asStruct()).isEqualTo(expectedDepthTwo.asStruct());
}
+ @Test
+ public void testSelectInIdOrder() {
+ Schema schema =
+ new Schema(
+ required(1, "id", Types.IntegerType.get()),
+ required(3, "b", Types.IntegerType.get()),
+ required(2, "a", Types.IntegerType.get()));
+
+ Schema result = TypeUtil.selectInIdOrder(schema, Sets.newHashSet(2, 3));
+
+ assertThat(result.columns()).hasSize(2);
+ assertThat(result.columns().get(0).fieldId()).isEqualTo(2);
+ assertThat(result.columns().get(1).fieldId()).isEqualTo(3);
+
+ // verify that different input orderings produce the same result
+ Schema schemaReversed =
+ new Schema(
+ required(2, "a", Types.IntegerType.get()),
+ required(3, "b", Types.IntegerType.get()),
+ required(1, "id", Types.IntegerType.get()));
+
+ Schema resultReversed = TypeUtil.selectInIdOrder(schemaReversed, Sets.newHashSet(2, 3));
+ assertThat(resultReversed.asStruct()).isEqualTo(result.asStruct());
+ }
+
@Test
public void testProjectMap() {
// We can't partially project keys because it changes key equality
diff --git a/api/src/test/java/org/apache/iceberg/util/TestCharSequenceSet.java b/api/src/test/java/org/apache/iceberg/util/TestCharSequenceSet.java
index 324742c07a2d..093d2a0c6b87 100644
--- a/api/src/test/java/org/apache/iceberg/util/TestCharSequenceSet.java
+++ b/api/src/test/java/org/apache/iceberg/util/TestCharSequenceSet.java
@@ -19,10 +19,12 @@
package org.apache.iceberg.util;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.Arrays;
import java.util.Collections;
import java.util.Set;
+import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
import org.junit.jupiter.api.Test;
@@ -42,15 +44,115 @@ public void testSearchingInCharSequenceCollection() {
@Test
public void nullString() {
- assertThat(CharSequenceSet.of(Arrays.asList((String) null))).contains((String) null);
+ assertThatThrownBy(() -> CharSequenceSet.of(Arrays.asList((String) null)))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid object: null");
assertThat(CharSequenceSet.empty()).doesNotContain((String) null);
}
+ @Test
+ public void emptySet() {
+ assertThat(CharSequenceSet.empty()).isEmpty();
+ assertThat(CharSequenceSet.empty()).doesNotContain("a", "b", "c");
+ }
+
+ @Test
+ public void insertionOrderIsMaintained() {
+ CharSequenceSet set = CharSequenceSet.empty();
+ set.addAll(ImmutableList.of("d", "a", "c"));
+ set.add("b");
+ set.add("d");
+
+ assertThat(set).hasSize(4).containsExactly("d", "a", "c", "b");
+ }
+
+ @Test
+ public void clear() {
+ CharSequenceSet set = CharSequenceSet.of(ImmutableList.of("a", "b"));
+ set.clear();
+ assertThat(set).isEmpty();
+ }
+
+ @Test
+ public void addAll() {
+ CharSequenceSet empty = CharSequenceSet.empty();
+ assertThatThrownBy(() -> empty.add(null))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid object: null");
+
+ assertThatThrownBy(() -> empty.addAll(null))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid collection: null");
+
+ assertThatThrownBy(() -> empty.addAll(Collections.singletonList(null)))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid object: null");
+
+ assertThatThrownBy(() -> empty.addAll(Arrays.asList("a", null)))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid object: null");
+
+ CharSequenceSet set = CharSequenceSet.empty();
+ set.addAll(ImmutableList.of("b", "a", "c", "a"));
+ assertThat(set).hasSize(3).containsExactly("b", "a", "c");
+ }
+
+ @Test
+ public void contains() {
+ CharSequenceSet set = CharSequenceSet.of(ImmutableList.of("b", "a"));
+ assertThatThrownBy(() -> set.contains(null))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid object: null");
+
+ assertThat(set).hasSize(2).containsExactly("b", "a").doesNotContain("c").doesNotContain("d");
+
+ assertThatThrownBy(() -> CharSequenceSet.of(Arrays.asList("c", "b", null, "a")))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid object: null");
+ }
+
+ @Test
+ public void containsAll() {
+ CharSequenceSet set = CharSequenceSet.of(ImmutableList.of("b", "a"));
+ assertThatThrownBy(() -> set.containsAll(null))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid collection: null");
+
+ assertThatThrownBy(() -> set.containsAll(Collections.singletonList(null)))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid object: null");
+
+ assertThatThrownBy(() -> set.containsAll(Arrays.asList("a", null)))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid object: null");
+
+ assertThat(set.containsAll(ImmutableList.of("a", "b"))).isTrue();
+ assertThat(set.containsAll(ImmutableList.of("b", "a", "c"))).isFalse();
+ assertThat(set.containsAll(ImmutableList.of("b"))).isTrue();
+ }
+
@Test
public void testRetainAll() {
+ CharSequenceSet empty = CharSequenceSet.empty();
+ assertThatThrownBy(() -> empty.retainAll(null))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid collection: null");
+
+ assertThatThrownBy(() -> empty.retainAll(Collections.singletonList(null)))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid object: null");
+
+ assertThatThrownBy(() -> empty.retainAll(Arrays.asList("123", null)))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid object: null");
+
+ assertThatThrownBy(() -> empty.retainAll(ImmutableList.of("456", "789", 123)))
+ .isInstanceOf(ClassCastException.class)
+ .hasMessage("Cannot cast java.lang.Integer to java.lang.CharSequence");
+
CharSequenceSet set = CharSequenceSet.of(ImmutableList.of("123", "456"));
- assertThat(set.retainAll(ImmutableList.of("456", "789", 123)))
+ assertThat(set.retainAll(ImmutableList.of("456", "789", "555")))
.overridingErrorMessage("Set should be changed")
.isTrue();
@@ -61,24 +163,74 @@ public void testRetainAll() {
.overridingErrorMessage("Set should not be changed")
.isFalse();
- assertThat(set.retainAll(ImmutableList.of(123, 456)))
+ assertThat(set.retainAll(ImmutableList.of("555", "789")))
.overridingErrorMessage("Set should be changed")
.isTrue();
assertThat(set).isEmpty();
}
+ @Test
+ public void toArray() {
+ CharSequenceSet set = CharSequenceSet.of(ImmutableList.of("b", "a"));
+ assertThat(set.toArray()).hasSize(2).containsExactly("b", "a");
+
+ CharSequence[] array = new CharSequence[1];
+ assertThat(set.toArray(array)).hasSize(2).containsExactly("b", "a");
+
+ array = new CharSequence[0];
+ assertThat(set.toArray(array)).hasSize(2).containsExactly("b", "a");
+
+ array = new CharSequence[5];
+ assertThat(set.toArray(array)).hasSize(5).containsExactly("b", "a", null, null, null);
+
+ array = new CharSequence[2];
+ assertThat(set.toArray(array)).hasSize(2).containsExactly("b", "a");
+ }
+
+ @Test
+ public void remove() {
+ CharSequenceSet set = CharSequenceSet.of(ImmutableSet.of("a", "b", "c"));
+ assertThatThrownBy(() -> set.remove(null))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid object: null");
+
+ set.remove("a");
+ assertThat(set).containsExactly("b", "c");
+ set.remove("b");
+ assertThat(set).containsExactly("c");
+ set.remove("c");
+ assertThat(set).isEmpty();
+ }
+
@Test
public void testRemoveAll() {
+ CharSequenceSet empty = CharSequenceSet.empty();
+ assertThatThrownBy(() -> empty.removeAll(null))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid collection: null");
+
+ assertThatThrownBy(() -> empty.removeAll(Collections.singletonList(null)))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid object: null");
+
+ assertThatThrownBy(() -> empty.removeAll(Arrays.asList("123", null)))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessage("Invalid object: null");
+
+ assertThatThrownBy(() -> empty.removeAll(ImmutableList.of("123", 456)))
+ .isInstanceOf(ClassCastException.class)
+ .hasMessage("Cannot cast java.lang.Integer to java.lang.CharSequence");
+
CharSequenceSet set = CharSequenceSet.of(ImmutableList.of("123", "456"));
- assertThat(set.removeAll(ImmutableList.of("456", "789", 123)))
+ assertThat(set.removeAll(ImmutableList.of("456", "789")))
.overridingErrorMessage("Set should be changed")
.isTrue();
assertThat(set).hasSize(1).contains("123");
set = CharSequenceSet.of(ImmutableList.of("123", "456"));
- assertThat(set.removeAll(ImmutableList.of(123, 456)))
+ assertThat(set.removeAll(ImmutableList.of("333", "789")))
.overridingErrorMessage("Set should not be changed")
.isFalse();
@@ -119,4 +271,17 @@ public void testEqualsAndHashCode() {
.isEqualTo(set3.hashCode())
.isEqualTo(set4.hashCode());
}
+
+ @Test
+ public void kryoSerialization() throws Exception {
+ CharSequenceSet charSequences = CharSequenceSet.of(ImmutableList.of("c", "b", "a"));
+ assertThat(TestHelpers.KryoHelpers.roundTripSerialize(charSequences)).isEqualTo(charSequences);
+ }
+
+ @Test
+ public void javaSerialization() throws Exception {
+ CharSequenceSet charSequences = CharSequenceSet.of(ImmutableList.of("c", "b", "a"));
+ CharSequenceSet deserialize = TestHelpers.deserialize(TestHelpers.serialize(charSequences));
+ assertThat(deserialize).isEqualTo(charSequences);
+ }
}
diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/ArrowFormatModels.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/ArrowFormatModels.java
new file mode 100644
index 000000000000..d70e12be7817
--- /dev/null
+++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/ArrowFormatModels.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.arrow.vectorized;
+
+import org.apache.arrow.vector.NullCheckingForGet;
+import org.apache.iceberg.formats.FormatModelRegistry;
+import org.apache.iceberg.parquet.ParquetFormatModel;
+
+public class ArrowFormatModels {
+ public static void register() {
+ FormatModelRegistry.register(
+ ParquetFormatModel.create(
+ ColumnarBatch.class,
+ Object.class,
+ (schema, fileSchema, engineSchema, idToConstant) ->
+ ArrowReader.VectorizedCombinedScanIterator.buildReader(
+ schema,
+ fileSchema,
+ NullCheckingForGet.NULL_CHECKING_ENABLED /* setArrowValidityVector */)));
+ }
+
+ private ArrowFormatModels() {}
+}
diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/ArrowReader.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/ArrowReader.java
index 06b7baec27d5..68a27bdfb8eb 100644
--- a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/ArrowReader.java
+++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/ArrowReader.java
@@ -29,7 +29,6 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
-import org.apache.arrow.vector.NullCheckingForGet;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.iceberg.CombinedScanTask;
@@ -40,13 +39,14 @@
import org.apache.iceberg.encryption.EncryptedFiles;
import org.apache.iceberg.encryption.EncryptedInputFile;
import org.apache.iceberg.encryption.EncryptionManager;
+import org.apache.iceberg.formats.FormatModelRegistry;
+import org.apache.iceberg.formats.ReadBuilder;
import org.apache.iceberg.io.CloseableGroup;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.io.CloseableIterator;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.mapping.NameMappingParser;
-import org.apache.iceberg.parquet.Parquet;
import org.apache.iceberg.parquet.TypeWithSchemaVisitor;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
@@ -189,8 +189,7 @@ public void close() throws IOException {
* Reads the data file and returns an iterator of {@link VectorSchemaRoot}. Only Parquet data file
* format is supported.
*/
- private static final class VectorizedCombinedScanIterator
- implements CloseableIterator {
+ static final class VectorizedCombinedScanIterator implements CloseableIterator {
private final Iterator fileItr;
private final Map inputFiles;
@@ -324,19 +323,8 @@ CloseableIterator open(FileScanTask task) {
InputFile location = getInputFile(task);
Preconditions.checkNotNull(location, "Could not find InputFile associated with FileScanTask");
if (task.file().format() == FileFormat.PARQUET) {
- Parquet.ReadBuilder builder =
- Parquet.read(location)
- .project(expectedSchema)
- .split(task.start(), task.length())
- .createBatchedReaderFunc(
- fileSchema ->
- buildReader(
- expectedSchema,
- fileSchema, /* setArrowValidityVector */
- NullCheckingForGet.NULL_CHECKING_ENABLED))
- .recordsPerBatch(batchSize)
- .filter(task.residual())
- .caseSensitive(caseSensitive);
+ ReadBuilder builder =
+ FormatModelRegistry.readBuilder(FileFormat.PARQUET, ColumnarBatch.class, location);
if (reuseContainers) {
builder.reuseContainers();
@@ -345,7 +333,14 @@ CloseableIterator open(FileScanTask task) {
builder.withNameMapping(NameMappingParser.fromJson(nameMapping));
}
- iter = builder.build();
+ iter =
+ builder
+ .project(expectedSchema)
+ .split(task.start(), task.length())
+ .recordsPerBatch(batchSize)
+ .caseSensitive(caseSensitive)
+ .filter(task.residual())
+ .build();
} else {
throw new UnsupportedOperationException(
"Format: " + task.file().format() + " not supported for batched reads");
@@ -376,7 +371,7 @@ private InputFile getInputFile(FileScanTask task) {
* @param fileSchema Schema of the data file.
* @param setArrowValidityVector Indicates whether to set the validity vector in Arrow vectors.
*/
- private static ArrowBatchReader buildReader(
+ static ArrowBatchReader buildReader(
Schema expectedSchema, MessageType fileSchema, boolean setArrowValidityVector) {
return (ArrowBatchReader)
TypeWithSchemaVisitor.visit(
diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorHolder.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorHolder.java
index 0245b6bba2d1..f8c0d6dd69b8 100644
--- a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorHolder.java
+++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorHolder.java
@@ -73,7 +73,7 @@ private VectorHolder(Types.NestedField field) {
icebergField = field;
}
- VectorHolder(FieldVector vec, Types.NestedField field, NullabilityHolder nulls) {
+ private VectorHolder(FieldVector vec, Types.NestedField field, NullabilityHolder nulls) {
columnDescriptor = null;
vector = vec;
isDictionaryEncoded = false;
diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedArrowReader.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedArrowReader.java
index 2cc7cde4541a..e9ebed2826f4 100644
--- a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedArrowReader.java
+++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedArrowReader.java
@@ -588,10 +588,18 @@ public Optional visit(
int bitWidth = intLogicalType.getBitWidth();
if (bitWidth == 8 || bitWidth == 16 || bitWidth == 32) {
+ // Iceberg has no unsigned integer type. Reading UINT32 into a 32-bit signed value would
+ // silently produce negative results for inputs above Integer.MAX_VALUE. UINT8 and UINT16
+ // both fit losslessly in a signed int32 and are allowed, matching the policy in
+ // BaseParquetReaders for the non-vectorized path.
+ Preconditions.checkArgument(
+ intLogicalType.isSigned() || bitWidth < 32, "Cannot read UINT32 as an int value");
((IntVector) vector).allocateNew(batchSize);
return Optional.of(
new LogicalTypeVisitorResult(vector, ReadType.INT, (int) IntVector.TYPE_WIDTH));
} else if (bitWidth == 64) {
+ Preconditions.checkArgument(
+ intLogicalType.isSigned(), "Cannot read UINT64 as a long value");
((BigIntVector) vector).allocateNew(batchSize);
return Optional.of(
new LogicalTypeVisitorResult(vector, ReadType.LONG, (int) BigIntVector.TYPE_WIDTH));
diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedReaderBuilder.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedReaderBuilder.java
index 15b55fb48d4a..3fbd797c26fb 100644
--- a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedReaderBuilder.java
+++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/VectorizedReaderBuilder.java
@@ -154,6 +154,16 @@ public VectorizedReader struct(
return null;
}
+ @Override
+ public VectorizedReader variant(
+ Types.VariantType iVariant, GroupType variant, VectorizedReader result) {
+ if (iVariant != null) {
+ throw new UnsupportedOperationException(
+ "Vectorized reads are not supported yet for variant fields");
+ }
+ return null;
+ }
+
@Override
public VectorizedReader primitive(
org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) {
diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedByteStreamSplitValuesReader.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedByteStreamSplitValuesReader.java
new file mode 100644
index 000000000000..58ea38231c4a
--- /dev/null
+++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedByteStreamSplitValuesReader.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.arrow.vectorized.parquet;
+
+import java.io.EOFException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.io.api.Binary;
+
+/**
+ * A {@link VectorizedValuesReader} implementation for the encoding type BYTE_STREAM_SPLIT. This is
+ * adapted from Parquet's ByteStreamSplitValuesReader.
+ *
+ * @see
+ * Parquet format encodings: BYTE_STREAM_SPLIT
+ */
+public class VectorizedByteStreamSplitValuesReader extends ValuesReader
+ implements VectorizedValuesReader {
+
+ private final int elementSizeInBytes;
+ private int totalBytesInStream;
+ private ByteBufferInputStream dataStream;
+ private ByteBuffer decodedDataStream;
+
+ public VectorizedByteStreamSplitValuesReader(int elementSizeInBytes) {
+ this.elementSizeInBytes = elementSizeInBytes;
+ }
+
+ @Override
+ public void initFromPage(int ignoredValueCount, ByteBufferInputStream in) {
+ this.totalBytesInStream = in.available();
+ this.dataStream = in;
+ }
+
+ @Override
+ public int readInteger() {
+ ensureDecoded();
+ return decodedDataStream.getInt();
+ }
+
+ @Override
+ public long readLong() {
+ ensureDecoded();
+ return decodedDataStream.getLong();
+ }
+
+ @Override
+ public float readFloat() {
+ ensureDecoded();
+ return decodedDataStream.getFloat();
+ }
+
+ @Override
+ public double readDouble() {
+ ensureDecoded();
+ return decodedDataStream.getDouble();
+ }
+
+ @Override
+ public Binary readBinary(int len) {
+ ensureDecoded();
+ byte[] bytes = new byte[len];
+ decodedDataStream.get(bytes);
+ return Binary.fromConstantByteArray(bytes);
+ }
+
+ @Override
+ public void readIntegers(int total, FieldVector vec, int rowId) {
+ readBatch(total, vec, rowId);
+ }
+
+ @Override
+ public void readLongs(int total, FieldVector vec, int rowId) {
+ readBatch(total, vec, rowId);
+ }
+
+ @Override
+ public void readFloats(int total, FieldVector vec, int rowId) {
+ readBatch(total, vec, rowId);
+ }
+
+ @Override
+ public void readDoubles(int total, FieldVector vec, int rowId) {
+ readBatch(total, vec, rowId);
+ }
+
+ @Override
+ public void skip() {
+ throw new UnsupportedOperationException("skip is not supported");
+ }
+
+ private void readBatch(int total, FieldVector vec, int rowId) {
+ ensureDecoded();
+ int bytesToRead = total * elementSizeInBytes;
+ long destOffset = (long) rowId * elementSizeInBytes;
+ ByteBuffer slice = decodedDataStream.slice();
+ slice.limit(bytesToRead);
+ vec.getDataBuffer().setBytes(destOffset, slice);
+ decodedDataStream.position(decodedDataStream.position() + bytesToRead);
+ }
+
+ private void ensureDecoded() {
+ if (decodedDataStream == null) {
+ Preconditions.checkState(
+ totalBytesInStream % elementSizeInBytes == 0,
+ "Stream size %s is not a multiple of element size %s",
+ totalBytesInStream,
+ elementSizeInBytes);
+ this.decodedDataStream = decode(totalBytesInStream / elementSizeInBytes);
+ }
+ }
+
+ private ByteBuffer decode(int valuesCount) {
+ ByteBuffer encoded;
+ try {
+ encoded = dataStream.slice(totalBytesInStream).slice();
+ } catch (EOFException e) {
+ throw new UncheckedIOException("Failed to read bytes from stream", e);
+ }
+ byte[] decoded = new byte[encoded.limit()];
+ int destByteIndex = 0;
+ for (int srcValueIndex = 0; srcValueIndex < valuesCount; srcValueIndex++) {
+ for (int stream = 0; stream < elementSizeInBytes; stream++, destByteIndex++) {
+ decoded[destByteIndex] = encoded.get(srcValueIndex + stream * valuesCount);
+ }
+ }
+ return ByteBuffer.wrap(decoded).order(ByteOrder.LITTLE_ENDIAN);
+ }
+}
diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedDeltaByteArrayValuesReader.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedDeltaByteArrayValuesReader.java
new file mode 100644
index 000000000000..6972a22fd9c1
--- /dev/null
+++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedDeltaByteArrayValuesReader.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.arrow.vectorized.parquet;
+
+import java.io.IOException;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.io.api.Binary;
+
+/**
+ * A {@link VectorizedValuesReader} implementation for DELTA_BYTE_ARRAY encoding. This encoding
+ * stores delta-encoded prefix lengths followed by suffixes encoded as DELTA_LENGTH_BYTE_ARRAY. Each
+ * value is reconstructed by taking the prefix of the previous value and appending the suffix. This
+ * is adapted from Spark's VectorizedDeltaByteArrayReader.
+ *
+ * @see
+ * Parquet format encodings: DELTA_BYTE_ARRAY
+ */
+public class VectorizedDeltaByteArrayValuesReader extends ValuesReader
+ implements VectorizedValuesReader {
+
+ private int[] prefixLengths;
+ private VectorizedDeltaLengthByteArrayValuesReader suffixReader;
+ private Binary previous = Binary.EMPTY;
+ private int currentRow = 0;
+
+ @Override
+ public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOException {
+ VectorizedDeltaEncodedValuesReader prefixLengthReader =
+ new VectorizedDeltaEncodedValuesReader();
+ prefixLengthReader.initFromPage(valueCount, in);
+ this.prefixLengths = prefixLengthReader.readIntegers(prefixLengthReader.totalValueCount(), 0);
+ this.suffixReader = new VectorizedDeltaLengthByteArrayValuesReader();
+ this.suffixReader.initFromPage(valueCount, in);
+ }
+
+ @Override
+ public int readInteger() {
+ return prefixLengths[currentRow] + suffixReader.lengthForCurrentRow();
+ }
+
+ @Override
+ public Binary readBinary(int len) {
+ int prefixLength = prefixLengths[currentRow];
+ Binary suffix = suffixReader.readBinary(len - prefixLength);
+ this.currentRow++;
+
+ if (prefixLength == 0) {
+ this.previous = suffix;
+ return suffix;
+ }
+
+ byte[] prefixBytes = previous.getBytes();
+ byte[] suffixBytes = suffix.getBytes();
+ byte[] out = new byte[prefixLength + suffixBytes.length];
+ System.arraycopy(prefixBytes, 0, out, 0, prefixLength);
+ System.arraycopy(suffixBytes, 0, out, prefixLength, suffixBytes.length);
+ this.previous = Binary.fromConstantByteArray(out);
+ return previous;
+ }
+
+ @Override
+ public void skip() {
+ throw new UnsupportedOperationException("skip is not supported");
+ }
+}
diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedDeltaEncodedValuesReader.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedDeltaEncodedValuesReader.java
index 115518e1fb50..446b83c7f10d 100644
--- a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedDeltaEncodedValuesReader.java
+++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedDeltaEncodedValuesReader.java
@@ -29,7 +29,6 @@
import org.apache.parquet.column.values.bitpacking.BytePackerForLong;
import org.apache.parquet.column.values.bitpacking.Packer;
import org.apache.parquet.io.ParquetDecodingException;
-import org.apache.parquet.io.api.Binary;
/**
* A {@link VectorizedValuesReader} implementation for the encoding type DELTA_BINARY_PACKED. This
@@ -91,18 +90,6 @@ public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOExce
firstValue = BytesUtils.readZigZagVarLong(this.inputStream);
}
- /** DELTA_BINARY_PACKED only supports INT32 and INT64 */
- @Override
- public byte readByte() {
- throw new UnsupportedOperationException("readByte is not supported");
- }
-
- /** DELTA_BINARY_PACKED only supports INT32 and INT64 */
- @Override
- public short readShort() {
- throw new UnsupportedOperationException("readShort is not supported");
- }
-
@Override
public int readInteger() {
readValues(1, null, 0, INT_SIZE, (f, i, v) -> intVal = (int) v);
@@ -121,10 +108,8 @@ public void skip() {
throw new UnsupportedOperationException("skip is not supported");
}
- /** DELTA_BINARY_PACKED only supports INT32 and INT64 */
- @Override
- public Binary readBinary(int len) {
- throw new UnsupportedOperationException("readBinary is not supported");
+ int totalValueCount() {
+ return totalValueCount;
}
@Override
@@ -132,21 +117,20 @@ public void readIntegers(int total, FieldVector vec, int rowId) {
readValues(total, vec, rowId, INT_SIZE, (f, i, v) -> f.getDataBuffer().setInt(i, (int) v));
}
- @Override
- public void readLongs(int total, FieldVector vec, int rowId) {
- readValues(total, vec, rowId, LONG_SIZE, (f, i, v) -> f.getDataBuffer().setLong(i, v));
+ int[] readIntegers(int total, int rowId) {
+ int[] result = new int[total];
+ readValues(
+ total,
+ null,
+ rowId,
+ INT_SIZE,
+ (vec, idx, val) -> result[(int) (idx / INT_SIZE)] = (int) val);
+ return result;
}
- /** DELTA_BINARY_PACKED only supports INT32 and INT64 */
@Override
- public void readFloats(int total, FieldVector vec, int rowId) {
- throw new UnsupportedOperationException("readFloats is not supported");
- }
-
- /** DELTA_BINARY_PACKED only supports INT32 and INT64 */
- @Override
- public void readDoubles(int total, FieldVector vec, int rowId) {
- throw new UnsupportedOperationException("readDoubles is not supported");
+ public void readLongs(int total, FieldVector vec, int rowId) {
+ readValues(total, vec, rowId, LONG_SIZE, (f, i, v) -> f.getDataBuffer().setLong(i, v));
}
private void readValues(
diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedDeltaLengthByteArrayValuesReader.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedDeltaLengthByteArrayValuesReader.java
new file mode 100644
index 000000000000..9cec5b6e62bf
--- /dev/null
+++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedDeltaLengthByteArrayValuesReader.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.arrow.vectorized.parquet;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
+import org.apache.parquet.bytes.ByteBufferInputStream;
+import org.apache.parquet.column.values.ValuesReader;
+import org.apache.parquet.io.api.Binary;
+
+/**
+ * A {@link VectorizedValuesReader} implementation for DELTA_LENGTH_BYTE_ARRAY encoding. This
+ * encoding stores delta-encoded byte array lengths followed by the concatenated byte array data.
+ * This is adapted from Spark's VectorizedDeltaLengthByteArrayReader.
+ *
+ * @see
+ * Parquet format encodings: DELTA_LENGTH_BYTE_ARRAY
+ */
+public class VectorizedDeltaLengthByteArrayValuesReader extends ValuesReader
+ implements VectorizedValuesReader {
+
+ private ByteBufferInputStream dataStream;
+ private int[] lengths;
+ private int currentRow = 0;
+
+ @Override
+ public void initFromPage(int valueCount, ByteBufferInputStream in) throws IOException {
+ VectorizedDeltaEncodedValuesReader lengthReader = new VectorizedDeltaEncodedValuesReader();
+ lengthReader.initFromPage(valueCount, in);
+ this.lengths = lengthReader.readIntegers(lengthReader.totalValueCount(), 0);
+ this.dataStream = in;
+ }
+
+ int lengthForCurrentRow() {
+ return lengths[currentRow];
+ }
+
+ @Override
+ public Binary readBinary(int len) {
+ try {
+ ByteBuffer buffer = dataStream.slice(len);
+ this.currentRow++;
+ if (buffer.hasArray()) {
+ return Binary.fromConstantByteArray(
+ buffer.array(), buffer.arrayOffset() + buffer.position(), len);
+ } else {
+ byte[] bytes = new byte[len];
+ buffer.get(bytes);
+ return Binary.fromConstantByteArray(bytes);
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to read binary data", e);
+ }
+ }
+
+ @Override
+ public int readInteger() {
+ return lengths[currentRow];
+ }
+
+ @Override
+ public void skip() {
+ throw new UnsupportedOperationException("skip is not supported");
+ }
+}
diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedPageIterator.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedPageIterator.java
index be1a3324ae43..4a06f64b5af9 100644
--- a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedPageIterator.java
+++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedPageIterator.java
@@ -34,6 +34,7 @@
import org.apache.parquet.column.values.RequiresPreviousReader;
import org.apache.parquet.column.values.ValuesReader;
import org.apache.parquet.io.ParquetDecodingException;
+import org.apache.parquet.schema.PrimitiveType;
public class VectorizedPageIterator extends BasePageIterator {
private final boolean setArrowValidityVector;
@@ -100,6 +101,17 @@ protected void initDataReader(Encoding dataEncoding, ByteBufferInputStream in, i
case DELTA_BINARY_PACKED:
valuesReader = new VectorizedDeltaEncodedValuesReader();
break;
+ case DELTA_LENGTH_BYTE_ARRAY:
+ valuesReader = new VectorizedDeltaLengthByteArrayValuesReader();
+ break;
+ case DELTA_BYTE_ARRAY:
+ valuesReader = new VectorizedDeltaByteArrayValuesReader();
+ break;
+ case BYTE_STREAM_SPLIT:
+ valuesReader =
+ new VectorizedByteStreamSplitValuesReader(
+ byteStreamSplitElementSize(desc.getPrimitiveType()));
+ break;
default:
throw new UnsupportedOperationException(
"Cannot support vectorized reads for column "
@@ -371,6 +383,22 @@ protected void nextDictEncodedVal(
}
}
+ private static int byteStreamSplitElementSize(PrimitiveType type) {
+ switch (type.getPrimitiveTypeName()) {
+ case INT32:
+ case FLOAT:
+ return VectorizedValuesReader.INT_SIZE;
+ case INT64:
+ case DOUBLE:
+ return VectorizedValuesReader.LONG_SIZE;
+ case FIXED_LEN_BYTE_ARRAY:
+ return type.getTypeLength();
+ default:
+ throw new UnsupportedOperationException(
+ "Byte stream split encoding is not supported for type " + type.getPrimitiveTypeName());
+ }
+ }
+
private int getActualBatchSize(int expectedBatchSize) {
return Math.min(expectedBatchSize, triplesCount - triplesRead);
}
diff --git a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedValuesReader.java b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedValuesReader.java
index 7c23149b18ab..85216dd94af0 100644
--- a/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedValuesReader.java
+++ b/arrow/src/main/java/org/apache/iceberg/arrow/vectorized/parquet/VectorizedValuesReader.java
@@ -37,44 +37,68 @@ interface VectorizedValuesReader {
int DOUBLE_SIZE = 8;
/** Read a single boolean */
- boolean readBoolean();
+ default boolean readBoolean() {
+ throw new UnsupportedOperationException("readBoolean is not supported");
+ }
/** Read a single byte */
- byte readByte();
+ default byte readByte() {
+ throw new UnsupportedOperationException("readByte is not supported");
+ }
/** Read a single short */
- short readShort();
+ default short readShort() {
+ throw new UnsupportedOperationException("readShort is not supported");
+ }
/** Read a single integer */
- int readInteger();
+ default int readInteger() {
+ throw new UnsupportedOperationException("readInteger is not supported");
+ }
/** Read a single long */
- long readLong();
+ default long readLong() {
+ throw new UnsupportedOperationException("readLong is not supported");
+ }
/** Read a single float */
- float readFloat();
+ default float readFloat() {
+ throw new UnsupportedOperationException("readFloat is not supported");
+ }
/** Read a single double */
- double readDouble();
+ default double readDouble() {
+ throw new UnsupportedOperationException("readDouble is not supported");
+ }
/**
* Read binary data of some length
*
* @param len The number of bytes to read
*/
- Binary readBinary(int len);
+ default Binary readBinary(int len) {
+ throw new UnsupportedOperationException("readBinary is not supported");
+ }
/** Read `total` integers into `vec` starting at `vec[rowId]` */
- void readIntegers(int total, FieldVector vec, int rowId);
+ default void readIntegers(int total, FieldVector vec, int rowId) {
+ throw new UnsupportedOperationException("readIntegers is not supported");
+ }
/** Read `total` longs into `vec` starting at `vec[rowId]` */
- void readLongs(int total, FieldVector vec, int rowId);
+ default void readLongs(int total, FieldVector vec, int rowId) {
+ throw new UnsupportedOperationException("readLongs is not supported");
+ }
/** Read `total` floats into `vec` starting at `vec[rowId]` */
- void readFloats(int total, FieldVector vec, int rowId);
+ default void readFloats(int total, FieldVector vec, int rowId) {
+ throw new UnsupportedOperationException("readFloats is not supported");
+ }
/** Read `total` doubles into `vec` starting at `vec[rowId]` */
- void readDoubles(int total, FieldVector vec, int rowId);
+ default void readDoubles(int total, FieldVector vec, int rowId) {
+ throw new UnsupportedOperationException("readDoubles is not supported");
+ }
/**
* Initialize the reader from a page. See {@link ValuesReader#initFromPage(int,
diff --git a/arrow/src/test/java/org/apache/iceberg/arrow/vectorized/TestArrowReader.java b/arrow/src/test/java/org/apache/iceberg/arrow/vectorized/TestArrowReader.java
index 34e83de15207..cf3eb2700265 100644
--- a/arrow/src/test/java/org/apache/iceberg/arrow/vectorized/TestArrowReader.java
+++ b/arrow/src/test/java/org/apache/iceberg/arrow/vectorized/TestArrowReader.java
@@ -21,6 +21,7 @@
import static org.apache.iceberg.Files.localInput;
import static org.apache.parquet.schema.Types.primitive;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.io.File;
import java.io.IOException;
@@ -41,6 +42,7 @@
import java.util.concurrent.TimeUnit;
import java.util.function.BiFunction;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.BitVector;
import org.apache.arrow.vector.DateDayVector;
@@ -101,6 +103,9 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
/**
* Test cases for {@link ArrowReader}.
@@ -383,6 +388,111 @@ public void testTimestampMillisAreReadCorrectly() throws Exception {
assertThat(totalRowsRead).as("Should read all rows").isEqualTo(millisValues.size());
}
+ @ParameterizedTest
+ @MethodSource("rejectedUnsignedIntegerCases")
+ public void testUnsignedIntegerColumnThrowsException(
+ int unsignedBitWidth,
+ PrimitiveType.PrimitiveTypeName physicalType,
+ Schema schema,
+ String expectedMessage)
+ throws Exception {
+ Table table = createSingleRowUnsignedIntTable(schema, physicalType, unsignedBitWidth, 100L);
+
+ assertThatThrownBy(
+ () -> {
+ try (VectorizedTableScanIterable vectorizedReader =
+ new VectorizedTableScanIterable(table.newScan(), 1024, false)) {
+ for (ColumnarBatch batch : vectorizedReader) {
+ batch.createVectorSchemaRootFromVectors().close();
+ }
+ }
+ })
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(expectedMessage);
+ }
+
+ @ParameterizedTest
+ @MethodSource("acceptedUnsignedSmallIntegerCases")
+ public void testUnsignedSmallIntegerColumnRoundtrips(int unsignedBitWidth, int value)
+ throws Exception {
+ Schema schema = new Schema(Types.NestedField.optional(1, "col", Types.IntegerType.get()));
+ Table table =
+ createSingleRowUnsignedIntTable(
+ schema, PrimitiveType.PrimitiveTypeName.INT32, unsignedBitWidth, value);
+
+ int totalRows = 0;
+ try (VectorizedTableScanIterable vectorizedReader =
+ new VectorizedTableScanIterable(table.newScan(), 1024, false)) {
+ for (ColumnarBatch batch : vectorizedReader) {
+ VectorSchemaRoot root = batch.createVectorSchemaRootFromVectors();
+ assertThat(((IntVector) root.getVector("col")).get(0))
+ .as("UINT%d value should round-trip through int", unsignedBitWidth)
+ .isEqualTo(value);
+ totalRows += root.getRowCount();
+ root.close();
+ }
+ }
+
+ assertThat(totalRows).isEqualTo(1);
+ }
+
+ private static Stream rejectedUnsignedIntegerCases() {
+ return Stream.of(
+ Arguments.of(
+ 32,
+ PrimitiveType.PrimitiveTypeName.INT32,
+ new Schema(Types.NestedField.optional(1, "col", Types.IntegerType.get())),
+ "Cannot read UINT32 as an int value"),
+ Arguments.of(
+ 64,
+ PrimitiveType.PrimitiveTypeName.INT64,
+ new Schema(Types.NestedField.optional(1, "col", Types.LongType.get())),
+ "Cannot read UINT64 as a long value"));
+ }
+
+ private static Stream acceptedUnsignedSmallIntegerCases() {
+ return Stream.of(Arguments.of(8, 250), Arguments.of(16, 50000));
+ }
+
+ private Table createSingleRowUnsignedIntTable(
+ Schema schema, PrimitiveType.PrimitiveTypeName physicalType, int unsignedBitWidth, long value)
+ throws IOException {
+ tables = new HadoopTables();
+ Table table = tables.create(schema, tempDir.toURI() + "/uint" + unsignedBitWidth);
+
+ MessageType parquetSchema =
+ new MessageType(
+ "test",
+ primitive(physicalType, Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.intType(unsignedBitWidth, false))
+ .id(1)
+ .named("col"));
+
+ File testFile =
+ new File(tempDir, "unsigned-int" + unsignedBitWidth + "-" + System.nanoTime() + ".parquet");
+ try (ParquetWriter writer =
+ ExampleParquetWriter.builder(new Path(testFile.toURI())).withType(parquetSchema).build()) {
+ SimpleGroupFactory factory = new SimpleGroupFactory(parquetSchema);
+ Group group = factory.newGroup();
+ if (physicalType == PrimitiveType.PrimitiveTypeName.INT64) {
+ group.add("col", value);
+ } else {
+ group.add("col", (int) value);
+ }
+ writer.write(group);
+ }
+
+ DataFile dataFile =
+ DataFiles.builder(PartitionSpec.unpartitioned())
+ .withPath(testFile.getAbsolutePath())
+ .withFileSizeInBytes(testFile.length())
+ .withFormat(FileFormat.PARQUET)
+ .withRecordCount(1)
+ .build();
+ table.newAppend().appendFile(dataFile).commit();
+ return table;
+ }
+
/**
* Run the following verifications:
*
diff --git a/arrow/src/test/java/org/apache/iceberg/arrow/vectorized/TestVectorizedReaderBuilder.java b/arrow/src/test/java/org/apache/iceberg/arrow/vectorized/TestVectorizedReaderBuilder.java
new file mode 100644
index 000000000000..e3d76515bcc7
--- /dev/null
+++ b/arrow/src/test/java/org/apache/iceberg/arrow/vectorized/TestVectorizedReaderBuilder.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.arrow.vectorized;
+
+import static org.assertj.core.api.Assertions.assertThatNoException;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.parquet.TypeWithSchemaVisitor;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types.IntegerType;
+import org.apache.iceberg.types.Types.NestedField;
+import org.apache.iceberg.types.Types.VariantType;
+import org.apache.iceberg.variants.Variant;
+import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
+import org.apache.parquet.schema.Type;
+import org.apache.parquet.schema.Types;
+import org.junit.jupiter.api.Test;
+
+public class TestVectorizedReaderBuilder {
+
+ @Test
+ public void testVariantNotSupportedInVectorizedReads() {
+ Schema icebergSchema =
+ new Schema(
+ NestedField.required(1, "id", IntegerType.get()),
+ NestedField.optional(2, "data", VariantType.get()));
+
+ MessageType parquetSchema = parquetSchemaWithVariant();
+
+ VectorizedReaderBuilder builder =
+ new VectorizedReaderBuilder(
+ icebergSchema, parquetSchema, false, ImmutableMap.of(), readers -> null);
+
+ assertThatThrownBy(
+ () -> TypeWithSchemaVisitor.visit(icebergSchema.asStruct(), parquetSchema, builder))
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining("Vectorized reads are not supported yet for variant fields");
+ }
+
+ @Test
+ public void testVariantSkippedWhenNotInProjection() {
+ Schema icebergSchema = new Schema(NestedField.required(1, "id", IntegerType.get()));
+
+ MessageType parquetSchema = parquetSchemaWithVariant();
+
+ VectorizedReaderBuilder builder =
+ new VectorizedReaderBuilder(
+ icebergSchema, parquetSchema, false, ImmutableMap.of(), readers -> null);
+
+ assertThatNoException()
+ .describedAs("Variant not in projection should not throw")
+ .isThrownBy(
+ () -> TypeWithSchemaVisitor.visit(icebergSchema.asStruct(), parquetSchema, builder));
+ }
+
+ private static MessageType parquetSchemaWithVariant() {
+ return Types.buildMessage()
+ .addField(
+ Types.primitive(PrimitiveTypeName.INT32, Type.Repetition.REQUIRED).id(1).named("id"))
+ .addField(
+ Types.buildGroup(Type.Repetition.OPTIONAL)
+ .as(LogicalTypeAnnotation.variantType(Variant.VARIANT_SPEC_VERSION))
+ .addField(
+ Types.primitive(PrimitiveTypeName.BINARY, Type.Repetition.REQUIRED)
+ .named("metadata"))
+ .addField(
+ Types.primitive(PrimitiveTypeName.BINARY, Type.Repetition.REQUIRED)
+ .named("value"))
+ .id(2)
+ .named("data"))
+ .named("table");
+ }
+}
diff --git a/aws-bundle/LICENSE b/aws-bundle/LICENSE
index e720d50b8d6a..d8484c933f9e 100644
--- a/aws-bundle/LICENSE
+++ b/aws-bundle/LICENSE
@@ -203,326 +203,636 @@
--------------------------------------------------------------------------------
-This binary artifact contains code from the following projects:
+This product bundles Apache Commons Codec.
---------------------------------------------------------------------------------
-
-Group: commons-codec Name: commons-codec Version: 1.17.1
Project URL: https://commons.apache.org/proper/commons-codec/
-License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt
-
---------------------------------------------------------------------------------
-
-Group: commons-logging Name: commons-logging Version: 1.2
-Project URL: http://commons.apache.org/proper/commons-logging/
-License: The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt
-
---------------------------------------------------------------------------------
-
-Group: io.netty Name: netty-buffer Version: 4.1.124.Final
-Project URL: https://netty.io/
License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-codec Version: 4.1.124.Final
-Project URL: https://netty.io/
-License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+This product bundles Apache Commons Logging.
---------------------------------------------------------------------------------
-
-Group: io.netty Name: netty-codec-http Version: 4.1.124.Final
-Project URL: https://netty.io/
+Project URL: http://commons.apache.org/proper/commons-logging/
License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-codec-http2 Version: 4.1.124.Final
-Project URL: https://netty.io/
-License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
+This product bundles Apache Parquet (bundled by AWS Analytics Accelerator S3).
-Group: io.netty Name: netty-common Version: 4.1.124.Final
-Project URL: https://netty.io/
+Copyright: 2014-2024 The Apache Software Foundation
+Project URL: https://parquet.apache.org/
License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-handler Version: 4.1.124.Final
-Project URL: https://netty.io/
-License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
+This product bundles Apache Thrift (bundled by Parquet).
-Group: io.netty Name: netty-resolver Version: 4.1.124.Final
-Project URL: https://netty.io/
+Copyright: 2006-2017 The Apache Software Foundation.
+Project URL: https://thrift.apache.org/
License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-transport Version: 4.1.124.Final
-Project URL: https://netty.io/
-License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
+This product bundles Netty.
-Group: io.netty Name: netty-transport-classes-epoll Version: 4.1.124.Final
Project URL: https://netty.io/
License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: io.netty Name: netty-transport-native-unix-common Version: 4.1.124.Final
-Project URL: https://netty.io/
-License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
+This product bundles Apache HttpComponents (core and client).
---------------------------------------------------------------------------------
-
-Group: org.apache.httpcomponents Name: httpclient Version: 4.5.13
-Project URL: http://hc.apache.org/httpcomponents-client
-License: Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt
+Project URL: https://hc.apache.org
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: org.apache.httpcomponents Name: httpcore Version: 4.4.16
-Project URL: http://hc.apache.org/httpcomponents-core-ga
-License: Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt
+This product bundles Reactive Streams.
---------------------------------------------------------------------------------
-
-Group: org.reactivestreams Name: reactive-streams Version: 1.0.4
Project URL: http://reactive-streams.org
-License: CC0 - http://creativecommons.org/publicdomain/zero/1.0/
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: annotations Version: 2.33.0
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+License: MIT-0
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: apache-client Version: 2.33.0
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: arns Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: auth Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: aws-core Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: aws-json-protocol Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: aws-query-protocol Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: aws-xml-protocol Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: checksums Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+| Copyright 2014 Reactive Streams
+|
+| Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so.
+|
+| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: checksums-spi Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: crt-core Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+This product bundles AWS SDK for Java.
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: dynamodb Version: 2.33.0
Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: endpoints-spi Version: 2.33.0
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: glue Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: http-auth Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: http-auth-aws Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: http-auth-aws-crt Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: http-auth-aws-eventstream Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: http-auth-spi Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: http-client-spi Version: 2.33.0
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: iam Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: identity-spi Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
+This product bundles AWS SDK CRT for Java.
-Group: software.amazon.awssdk Name: json-utils Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+Project URL: https://github.com/awslabs/aws-crt-java
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: kms Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+This product bundles AWS EventStream for Java.
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: lakeformation Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+Project URL: https://github.com/awslabs/aws-eventstream-java
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: metrics-spi Version: 2.33.0
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
+This product bundles AWS AccessGrants Plugin for Java.
-Group: software.amazon.awssdk Name: netty-nio-client Version: 2.33.0
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+Project URL: https://github.com/aws/aws-s3-accessgrants-plugin-java-v2
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: profiles Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
+This product bundles AWS Analytics Accelerator S3.
-Group: software.amazon.awssdk Name: protocol-core Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+Project URL: https://github.com/awslabs/analytics-accelerator-s3
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: regions Version: 2.33.0
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
+This product bundles Caffeine by Ben Manes.
-Group: software.amazon.awssdk Name: s3 Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+Copyright: 2014-2020 Ben Manes and contributors
+Project URL: https://github.com/ben-manes/caffeine
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: sdk-core Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+This product bundles failsafe (bundled by AWS Analytics Accelerator S3).
---------------------------------------------------------------------------------
-
-Group: software.amazon.awssdk Name: sso Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+Copyright: Jonathan Halterman and friends
+Project URL: https://failsafe.dev/
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: sts Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
+This product bundles Jackson JSON Processor.
-Group: software.amazon.awssdk Name: third-party-jackson-core Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+Copyright: 2007-2020 Tatu Saloranta and other contributors
+Project URL: https://github.com/FasterXML/jackson
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk Name: utils Version: 2.33.0
-Project URL: https://aws.amazon.com/sdkforjava
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+This product bundles checkerframework checker-qual.
+
+Copyright: 2004-2020 the Checker Framework developers
+Project URL: https://github.com/typetools/checker-framework
+License: MIT
+
+| The annotations are licensed under the MIT License. (The text of this
+| license appears below.) More specifically, all the parts of the Checker
+| Framework that you might want to include with your own program use the
+| MIT License. This is the checker-qual.jar file and all the files that
+| appear in it: every file in a qual/ directory, plus utility files such
+| as NullnessUtil.java, RegexUtil.java, SignednessUtil.java, etc.
+| In addition, the cleanroom implementations of third-party annotations,
+| which the Checker Framework recognizes as aliases for its own
+| annotations, are licensed under the MIT License.
+|
+| Permission is hereby granted, free of charge, to any person obtaining a copy
+| of this software and associated documentation files (the "Software"), to deal
+| in the Software without restriction, including without limitation the rights
+| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+| copies of the Software, and to permit persons to whom the Software is
+| furnished to do so, subject to the following conditions:
+|
+| The above copyright notice and this permission notice shall be included in
+| all copies or substantial portions of the Software.
+|
+| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+| THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This product bundles Google Error Prone Annotations.
+
+Copyright: Copyright 2011-2019 The Error Prone Authors
+Project URL: https://github.com/google/error-prone
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: software.amazon.awssdk.crt Name: aws-crt Version: 0.38.9
-Project URL: https://github.com/awslabs/aws-crt-java
-License: The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt
-
---------------------------------------------------------------------------------
+This product bundles JCTools (via Netty).
-Group: software.amazon.eventstream Name: eventstream Version: 1.0.1
-Project URL: https://github.com/awslabs/aws-eventstream-java
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+Project URL: https://github.com/JCTools/JCTools
+License: Apache License, Version 2.0 - https://www.apache.org/licenses/LICENSE-2.0
--------------------------------------------------------------------------------
-Group: software.amazon.s3.accessgrants Name: aws-s3-accessgrants-java-plugin Version: 2.3.0
-Project URL: https://github.com/aws/aws-s3-accessgrants-plugin-java-v2
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
-
---------------------------------------------------------------------------------
-Group: software.amazon.s3.analyticsaccelerator Name: analyticsaccelerator-s3 Version: 1.3.0
-Project URL: https://github.com/awslabs/analytics-accelerator-s3
-License: Apache License, Version 2.0 - https://aws.amazon.com/apache2.0
+This product bundles the Mozilla Public Suffix List (via Apache HttpComponents).
+
+Project URL: https://publicsuffix.org/
+License: Mozilla Public License, Version 2.0 - https://mozilla.org/MPL/2.0/
+
+| Mozilla Public License Version 2.0
+| ==================================
+|
+| 1. Definitions
+| --------------
+|
+| 1.1. "Contributor"
+| means each individual or legal entity that creates, contributes to
+| the creation of, or owns Covered Software.
+|
+| 1.2. "Contributor Version"
+| means the combination of the Contributions of others (if any) used
+| by a Contributor and that particular Contributor's Contribution.
+|
+| 1.3. "Contribution"
+| means Covered Software of a particular Contributor.
+|
+| 1.4. "Covered Software"
+| means Source Code Form to which the initial Contributor has attached
+| the notice in Exhibit A, the Executable Form of such Source Code
+| Form, and Modifications of such Source Code Form, in each case
+| including portions thereof.
+|
+| 1.5. "Incompatible With Secondary Licenses"
+| means
+|
+| (a) that the initial Contributor has attached the notice described
+| in Exhibit B to the Covered Software; or
+|
+| (b) that the Covered Software was made available under the terms of
+| version 1.1 or earlier of the License, but not also under the
+| terms of a Secondary License.
+|
+| 1.6. "Executable Form"
+| means any form of the work other than Source Code Form.
+|
+| 1.7. "Larger Work"
+| means a work that combines Covered Software with other material, in
+| a separate file or files, that is not Covered Software.
+|
+| 1.8. "License"
+| means this document.
+|
+| 1.9. "Licensable"
+| means having the right to grant, to the maximum extent possible,
+| whether at the time of the initial grant or subsequently, any and
+| all of the rights conveyed by this License.
+|
+| 1.10. "Modifications"
+| means any of the following:
+|
+| (a) any file in Source Code Form that results from an addition to,
+| deletion from, or modification of the contents of Covered
+| Software; or
+|
+| (b) any new file in Source Code Form that contains any Covered
+| Software.
+|
+| 1.11. "Patent Claims" of a Contributor
+| means any patent claim(s), including without limitation, method,
+| process, and apparatus claims, in any patent Licensable by such
+| Contributor that would be infringed, but for the grant of the
+| License, by the making, using, selling, offering for sale, having
+| made, import, or transfer of either its Contributions or its
+| Contributor Version.
+|
+| 1.12. "Secondary License"
+| means either the GNU General Public License, Version 2.0, the GNU
+| Lesser General Public License, Version 2.1, the GNU Affero General
+| Public License, Version 3.0, or any later versions of those
+| licenses.
+|
+| 1.13. "Source Code Form"
+| means the form of the work preferred for making modifications.
+|
+| 1.14. "You" (or "Your")
+| means an individual or a legal entity exercising rights under this
+| License. For legal entities, "You" includes any entity that
+| controls, is controlled by, or is under common control with You. For
+| purposes of this definition, "control" means (a) the power, direct
+| or indirect, to cause the direction or management of such entity,
+| whether by contract or otherwise, or (b) ownership of more than
+| fifty percent (50%) of the outstanding shares or beneficial
+| ownership of such entity.
+|
+| 2. License Grants and Conditions
+| --------------------------------
+|
+| 2.1. Grants
+|
+| Each Contributor hereby grants You a world-wide, royalty-free,
+| non-exclusive license:
+|
+| (a) under intellectual property rights (other than patent or trademark)
+| Licensable by such Contributor to use, reproduce, make available,
+| modify, display, perform, distribute, and otherwise exploit its
+| Contributions, either on an unmodified basis, with Modifications, or
+| as part of a Larger Work; and
+|
+| (b) under Patent Claims of such Contributor to make, use, sell, offer
+| for sale, have made, import, and otherwise transfer either its
+| Contributions or its Contributor Version.
+|
+| 2.2. Effective Date
+|
+| The licenses granted in Section 2.1 with respect to any Contribution
+| become effective for each Contribution on the date the Contributor first
+| distributes such Contribution.
+|
+| 2.3. Limitations on Grant Scope
+|
+| The licenses granted in this Section 2 are the only rights granted under
+| this License. No additional rights or licenses will be implied from the
+| distribution or licensing of Covered Software under this License.
+| Notwithstanding Section 2.1(b) above, no patent license is granted by a
+| Contributor:
+|
+| (a) for any code that a Contributor has removed from Covered Software;
+| or
+|
+| (b) for infringements caused by: (i) Your and any other third party's
+| modifications of Covered Software, or (ii) the combination of its
+| Contributions with other software (except as part of its Contributor
+| Version); or
+|
+| (c) under Patent Claims infringed by Covered Software in the absence of
+| its Contributions.
+|
+| This License does not grant any rights in the trademarks, service marks,
+| or logos of any Contributor (except as may be necessary to comply with
+| the notice requirements in Section 3.4).
+|
+| 2.4. Subsequent Licenses
+|
+| No Contributor makes additional grants as a result of Your choice to
+| distribute the Covered Software under a subsequent version of this
+| License (see Section 10.2) or under the terms of a Secondary License (if
+| permitted under the terms of Section 3.3).
+|
+| 2.5. Representation
+|
+| Each Contributor represents that the Contributor believes its
+| Contributions are its original creation(s) or it has sufficient rights
+| to grant the rights to its Contributions conveyed by this License.
+|
+| 2.6. Fair Use
+|
+| This License is not intended to limit any rights You have under
+| applicable copyright doctrines of fair use, fair dealing, or other
+| equivalents.
+|
+| 2.7. Conditions
+|
+| Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+| in Section 2.1.
+|
+| 3. Responsibilities
+| -------------------
+|
+| 3.1. Distribution of Source Form
+|
+| All distribution of Covered Software in Source Code Form, including any
+| Modifications that You create or to which You contribute, must be under
+| the terms of this License. You must inform recipients that the Source
+| Code Form of the Covered Software is governed by the terms of this
+| License, and how they can obtain a copy of this License. You may not
+| attempt to alter or restrict the recipients' rights in the Source Code
+| Form.
+|
+| 3.2. Distribution of Executable Form
+|
+| If You distribute Covered Software in Executable Form then:
+|
+| (a) such Covered Software must also be made available in Source Code
+| Form, as described in Section 3.1, and You must inform recipients of
+| the Executable Form how they can obtain a copy of such Source Code
+| Form by reasonable means in a timely manner, at a charge no more
+| than the cost of distribution to the recipient; and
+|
+| (b) You may distribute such Executable Form under the terms of this
+| License, or sublicense it under different terms, provided that the
+| license for the Executable Form does not attempt to limit or alter
+| the recipients' rights in the Source Code Form under this License.
+|
+| 3.3. Distribution of a Larger Work
+|
+| You may create and distribute a Larger Work under terms of Your choice,
+| provided that You also comply with the requirements of this License for
+| the Covered Software. If the Larger Work is a combination of Covered
+| Software with a work governed by one or more Secondary Licenses, and the
+| Covered Software is not Incompatible With Secondary Licenses, this
+| License permits You to additionally distribute such Covered Software
+| under the terms of such Secondary License(s), so that the recipient of
+| the Larger Work may, at their option, further distribute the Covered
+| Software under the terms of either this License or such Secondary
+| License(s).
+|
+| 3.4. Notices
+|
+| You may not remove or alter the substance of any license notices
+| (including copyright notices, patent notices, disclaimers of warranty,
+| or limitations of liability) contained within the Source Code Form of
+| the Covered Software, except that You may alter any license notices to
+| the extent required to remedy known factual inaccuracies.
+|
+| 3.5. Application of Additional Terms
+|
+| You may choose to offer, and to charge a fee for, warranty, support,
+| indemnity or liability obligations to one or more recipients of Covered
+| Software. However, You may do so only on Your own behalf, and not on
+| behalf of any Contributor. You must make it absolutely clear that any
+| such warranty, support, indemnity, or liability obligation is offered by
+| You alone, and You hereby agree to indemnify every Contributor for any
+| liability incurred by such Contributor as a result of warranty, support,
+| indemnity or liability terms You offer. You may include additional
+| disclaimers of warranty and limitations of liability specific to any
+| jurisdiction.
+|
+| 4. Inability to Comply Due to Statute or Regulation
+| ---------------------------------------------------
+|
+| If it is impossible for You to comply with any of the terms of this
+| License with respect to some or all of the Covered Software due to
+| statute, judicial order, or regulation then You must: (a) comply with
+| the terms of this License to the maximum extent possible; and (b)
+| describe the limitations and the code they affect. Such description must
+| be placed in a text file included with all distributions of the Covered
+| Software under this License. Except to the extent prohibited by statute
+| or regulation, such description must be sufficiently detailed for a
+| recipient of ordinary skill to be able to understand it.
+|
+| 5. Termination
+| --------------
+|
+| 5.1. The rights granted under this License will terminate automatically
+| if You fail to comply with any of its terms. However, if You become
+| compliant, then the rights granted under this License from a particular
+| Contributor are reinstated (a) provisionally, unless and until such
+| Contributor explicitly and finally terminates Your grants, and (b) on an
+| ongoing basis, if such Contributor fails to notify You of the
+| non-compliance by some reasonable means prior to 60 days after You have
+| come back into compliance. Moreover, Your grants from a particular
+| Contributor are reinstated on an ongoing basis if such Contributor
+| notifies You of the non-compliance by some reasonable means, this is the
+| first time You have received notice of non-compliance with this License
+| from such Contributor, and You become compliant prior to 30 days after
+| Your receipt of the notice.
+|
+| 5.2. If You initiate litigation against any entity by asserting a patent
+| infringement claim (excluding declaratory judgment actions,
+| counter-claims, and cross-claims) alleging that a Contributor Version
+| directly or indirectly infringes any patent, then the rights granted to
+| You by any and all Contributors for the Covered Software under Section
+| 2.1 of this License shall terminate.
+|
+| 5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+| end user license agreements (excluding distributors and resellers) which
+| have been validly granted by You or Your distributors under this License
+| prior to termination shall survive termination.
+|
+| ************************************************************************
+| * *
+| * 6. Disclaimer of Warranty *
+| * ------------------------- *
+| * *
+| * Covered Software is provided under this License on an "as is" *
+| * basis, without warranty of any kind, either expressed, implied, or *
+| * statutory, including, without limitation, warranties that the *
+| * Covered Software is free of defects, merchantable, fit for a *
+| * particular purpose or non-infringing. The entire risk as to the *
+| * quality and performance of the Covered Software is with You. *
+| * Should any Covered Software prove defective in any respect, You *
+| * (not any Contributor) assume the cost of any necessary servicing, *
+| * repair, or correction. This disclaimer of warranty constitutes an *
+| * essential part of this License. No use of any Covered Software is *
+| * authorized under this License except under this disclaimer. *
+| * *
+| ************************************************************************
+|
+| ************************************************************************
+| * *
+| * 7. Limitation of Liability *
+| * -------------------------- *
+| * *
+| * Under no circumstances and under no legal theory, whether tort *
+| * (including negligence), contract, or otherwise, shall any *
+| * Contributor, or anyone who distributes Covered Software as *
+| * permitted above, be liable to You for any direct, indirect, *
+| * special, incidental, or consequential damages of any character *
+| * including, without limitation, damages for lost profits, loss of *
+| * goodwill, work stoppage, computer failure or malfunction, or any *
+| * and all other commercial damages or losses, even if such party *
+| * shall have been informed of the possibility of such damages. This *
+| * limitation of liability shall not apply to liability for death or *
+| * personal injury resulting from such party's negligence to the *
+| * extent applicable law prohibits such limitation. Some *
+| * jurisdictions do not allow the exclusion or limitation of *
+| * incidental or consequential damages, so this exclusion and *
+| * limitation may not apply to You. *
+| * *
+| ************************************************************************
+|
+| 8. Litigation
+| -------------
+|
+| Any litigation relating to this License may be brought only in the
+| courts of a jurisdiction where the defendant maintains its principal
+| place of business and such litigation shall be governed by laws of that
+| jurisdiction, without reference to its conflict-of-law provisions.
+| Nothing in this Section shall prevent a party's ability to bring
+| cross-claims or counter-claims.
+|
+| 9. Miscellaneous
+| ----------------
+|
+| This License represents the complete agreement concerning the subject
+| matter hereof. If any provision of this License is held to be
+| unenforceable, such provision shall be reformed only to the extent
+| necessary to make it enforceable. Any law or regulation which provides
+| that the language of a contract shall be construed against the drafter
+| shall not be used to construe this License against a Contributor.
+|
+| 10. Versions of the License
+| ---------------------------
+|
+| 10.1. New Versions
+|
+| Mozilla Foundation is the license steward. Except as provided in Section
+| 10.3, no one other than the license steward has the right to modify or
+| publish new versions of this License. Each version will be given a
+| distinguishing version number.
+|
+| 10.2. Effect of New Versions
+|
+| You may distribute the Covered Software under the terms of the version
+| of the License under which You originally received the Covered Software,
+| or under the terms of any subsequent version published by the license
+| steward.
+|
+| 10.3. Modified Versions
+|
+| If you create software not governed by this License, and you want to
+| create a new license for such software, you may create and use a
+| modified version of this License if you rename the license and remove
+| any references to the name of the license steward (except to note that
+| such modified license differs from this License).
+|
+| 10.4. Distributing Source Code Form that is Incompatible With Secondary
+| Licenses
+|
+| If You choose to distribute Source Code Form that is Incompatible With
+| Secondary Licenses under the terms of this version of the License, the
+| notice described in Exhibit B of this License must be attached.
+|
+| Exhibit A - Source Code Form License Notice
+| -------------------------------------------
+|
+| This Source Code Form is subject to the terms of the Mozilla Public
+| License, v. 2.0. If a copy of the MPL was not distributed with this
+| file, You can obtain one at http://mozilla.org/MPL/2.0/.
+|
+| If it is not possible or desirable to put the notice in a particular
+| file, then You may include the notice in a location (such as a LICENSE
+| file in a relevant directory) where a recipient would be likely to look
+| for such a notice.
+|
+| You may add additional accurate notices of copyright ownership.
+|
+| Exhibit B - "Incompatible With Secondary Licenses" Notice
+| ---------------------------------------------------------
+|
+| This Source Code Form is "Incompatible With Secondary Licenses", as
+| defined by the Mozilla Public License, v. 2.0.
+
+--------------------------------------------------------------------------------
+
+This product bundles FastDoubleParser (via Jackson JSON Processor, via AWS SDK third-party-jackson-core).
+
+Copyright: 2023 Werner Randelshofer, Switzerland
+Project URL: https://github.com/wrandelshofer/FastDoubleParser
+License: MIT
+
+| Copyright (c) 2023 Werner Randelshofer, Switzerland
+|
+| Permission is hereby granted, free of charge, to any person obtaining a copy
+| of this software and associated documentation files (the "Software"), to deal
+| in the Software without restriction, including without limitation the rights
+| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+| copies of the Software, and to permit persons to whom the Software is
+| furnished to do so, subject to the following conditions:
+|
+| The above copyright notice and this permission notice shall be included in all
+| copies or substantial portions of the Software.
+|
+| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+| SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This product bundles fast_float (bundled by FastDoubleParser).
+
+Copyright: 2021 The fast_float authors
+Project URL: https://github.com/fastfloat/fast_float
+License: MIT
+
+| Copyright (c) 2021 The fast_float authors
+|
+| Permission is hereby granted, free of charge, to any person obtaining a copy
+| of this software and associated documentation files (the "Software"), to deal
+| in the Software without restriction, including without limitation the rights
+| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+| copies of the Software, and to permit persons to whom the Software is
+| furnished to do so, subject to the following conditions:
+|
+| The above copyright notice and this permission notice shall be included in all
+| copies or substantial portions of the Software.
+|
+| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+| SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This product bundles bigint (bundled by FastDoubleParser).
+
+Copyright: 2022 Tim Buktu
+Project URL: https://github.com/tbuktu/bigint
+License: BSD 2-Clause
+
+| Copyright 2022 Tim Buktu
+|
+| Redistribution and use in source and binary forms, with or without
+| modification, are permitted provided that the following conditions
+| are met:
+|
+| 1. Redistributions of source code must retain the above copyright notice, this
+| list of conditions and the following disclaimer.
+|
+| 2. Redistributions in binary form must reproduce the above copyright notice,
+| this list of conditions and the following disclaimer in the documentation
+| and/or other materials provided with the distribution.
+|
+| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+| ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+| WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+| DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+| FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+| DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+| SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+| CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+| OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/aws-bundle/NOTICE b/aws-bundle/NOTICE
index 20aa59905990..39738b74a297 100644
--- a/aws-bundle/NOTICE
+++ b/aws-bundle/NOTICE
@@ -1,106 +1,74 @@
Apache Iceberg
-Copyright 2017-2025 The Apache Software Foundation
+Copyright 2017-2026 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
--------------------------------------------------------------------------------
-NOTICE for Group: software.amazon.awssdk Name: annotations Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: apache-client Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: arns Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: auth Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: aws-core Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: aws-json-protocol Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: aws-query-protocol Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: aws-xml-protocol Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: checksums Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: checksums-spi Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: crt-core Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: dynamodb Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: endpoints-spi Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: glue Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: http-auth Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: http-auth-aws Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: http-auth-aws-crt Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: http-auth-aws-eventstream Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: http-auth-spi Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: http-client-spi Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: iam Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: identity-spi Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: json-utils Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: kms Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: lakeformation Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: metrics-spi Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: netty-nio-client Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: profiles Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: protocol-core Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: regions Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: retries Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: retries-spi Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: s3 Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: sdk-core Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: sso Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: sts Version: 2.33.0
-NOTICE for Group: software.amazon.awssdk Name: utils Version: 2.33.0
-NOTICE for Group: software.amazon.s3.accessgrants Name: aws-s3-accessgrants-java-plugin Version: 2.3.0
-NOTICE for Group: software.amazon.s3.analyticsaccelerator Name: analyticsaccelerator-s3 Version: 1.3.0
-
-AWS SDK for Java 2.0
-Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-
-This product includes software developed by
-Amazon Technologies, Inc (http://www.amazon.com/).
-
-**********************
-THIRD PARTY COMPONENTS
-**********************
-This software includes third party software subject to the following copyrights:
-- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty.
-- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc.
-- Apache Commons Lang - https://github.com/apache/commons-lang
-- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams
-- Jackson-core - https://github.com/FasterXML/jackson-core
-- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary
-
-The licenses for these third party components are included in LICENSE.txt
+This product bundles AWS SDK for Java with the following in its NOTICE file:
+|
+| AWS SDK for Java 2.0
+| Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+|
+| This product includes software developed by
+| Amazon Technologies, Inc ().
+|
+| **********************
+| THIRD PARTY COMPONENTS
+| **********************
+| This software includes third party software subject to the following copyrights:
+|
+| - XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty.
+| - PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc.
+| - Apache Commons Lang -
+| - Netty Reactive Streams -
+| - Jackson-core -
+| - Jackson-dataformat-cbor -
+|
+| The licenses for these third party components are included in LICENSE.txt
+|
--------------------------------------------------------------------------------
-NOTICE for Group: software.amazon.awssdk Name: third-party-jackson-core Version: 2.33.0
-
-# Jackson JSON processor
-
-Jackson is a high-performance, Free/Open Source JSON processing library.
-It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has
-been in development since 2007.
-It is currently developed by a community of developers.
-
-## Licensing
-
-Jackson 2.x core and extension components are licensed under Apache License 2.0
-To find the details that apply to this artifact see the accompanying LICENSE file.
-
-## Credits
-
-A list of contributors may be found from CREDITS(-2.x) file, which is included
-in some artifacts (usually source distributions); but is always available
-from the source code management (SCM) system project uses.
+This product bundles Jackson JSON Processor with the following in its NOTICE file:
+| # Jackson JSON processor
+|
+| Jackson is a high-performance, Free/Open Source JSON processing library.
+| It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has
+| been in development since 2007.
+| It is currently developed by a community of developers.
+|
+| ## Copyright
+|
+| Copyright 2007-, Tatu Saloranta (tatu.saloranta@iki.fi)
+|
+| ## Licensing
+|
+| Jackson 2.x core and extension components are licensed under Apache License 2.0
+| To find the details that apply to this artifact see the accompanying LICENSE file.
+|
+| ## Credits
+|
+| A list of contributors may be found from CREDITS(-2.x) file, which is included
+| in some artifacts (usually source distributions); but is always available
+| from the source code management (SCM) system project uses.
+|
+| ## FastDoubleParser
+|
+| jackson-core bundles a shaded copy of FastDoubleParser .
+| That code is available under an MIT license
+| under the following copyright.
+|
+| Copyright © 2023 Werner Randelshofer, Switzerland. MIT License.
+|
+| See FastDoubleParser-NOTICE for details of other source code included in FastDoubleParser
+| and the licenses and copyrights that apply to that code.
--------------------------------------------------------------------------------
-NOTICE for Group: io.netty Name: netty-buffer Version: 4.1.124.Final
-NOTICE for Group: io.netty Name: netty-codec Version: 4.1.124.Final
-NOTICE for Group: io.netty Name: netty-codec-http Version: 4.1.124.Final
-NOTICE for Group: io.netty Name: netty-codec-http2 Version: 4.1.124.Final
-NOTICE for Group: io.netty Name: netty-common Version: 4.1.124.Final
-NOTICE for Group: io.netty Name: netty-handler Version: 4.1.124.Final
-NOTICE for Group: io.netty Name: netty-resolver Version: 4.1.124.Final
-NOTICE for Group: io.netty Name: netty-transport Version: 4.1.124.Final
-NOTICE for Group: io.netty Name: netty-transport-classes-epoll Version: 4.1.124.Final
-NOTICE for Group: io.netty Name: netty-transport-native-unix-common Version: 4.1.124.Final
-
+This product bundles Netty with the following in its NOTICE file:
| The Netty Project
| =================
|
@@ -364,3 +332,8 @@ NOTICE for Group: io.netty Name: netty-transport-native-unix-common Version:
| * license/LICENSE.brotli4j.txt (Apache License 2.0)
| * HOMEPAGE:
| * https://github.com/hyperxpro/Brotli4j
+
+--------------------------------------------------------------------------------
+
+This product bundles AWS Analytics Accelerator S3 with the following in its NOTICE file:
+| Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
diff --git a/aws-bundle/build.gradle b/aws-bundle/build.gradle
index 5b9054812a50..541d5ae7a541 100644
--- a/aws-bundle/build.gradle
+++ b/aws-bundle/build.gradle
@@ -23,6 +23,14 @@ project(":iceberg-aws-bundle") {
tasks.jar.dependsOn tasks.shadowJar
+ configurations {
+ implementation {
+ exclude group: 'org.slf4j'
+ exclude group: 'org.apache.logging.slf4j'
+ exclude group: 'org.apache.logging.log4j'
+ }
+ }
+
dependencies {
implementation platform(libs.awssdk.bom)
implementation libs.awssdk.s3accessgrants
@@ -52,12 +60,6 @@ project(":iceberg-aws-bundle") {
include 'NOTICE'
}
- dependencies {
- exclude(dependency('org.slf4j:.*'))
- exclude(dependency('org.apache.logging.log4j:.*'))
- exclude(dependency('org.apache.logging.slf4j:.*'))
- }
-
// relocate AWS-specific versions
relocate 'org.apache.http', 'org.apache.iceberg.aws.shaded.org.apache.http'
relocate 'io.netty', 'org.apache.iceberg.aws.shaded.io.netty'
@@ -66,4 +68,6 @@ project(":iceberg-aws-bundle") {
jar {
enabled = false
}
+
+ apply from: "${rootDir}/runtime-deps.gradle"
}
diff --git a/aws-bundle/runtime-deps.txt b/aws-bundle/runtime-deps.txt
new file mode 100644
index 000000000000..b0c8cb946054
--- /dev/null
+++ b/aws-bundle/runtime-deps.txt
@@ -0,0 +1,70 @@
+com.github.ben-manes.caffeine:caffeine:2.9.3
+com.google.errorprone:error_prone_annotations:2.10.0
+commons-codec:commons-codec:1.17.1
+commons-logging:commons-logging:1.2
+io.netty:netty-buffer:4.2.13.Final
+io.netty:netty-codec-base:4.2.13.Final
+io.netty:netty-codec-compression:4.2.13.Final
+io.netty:netty-codec-http2:4.2.13.Final
+io.netty:netty-codec-http:4.2.13.Final
+io.netty:netty-codec-marshalling:4.2.13.Final
+io.netty:netty-codec-protobuf:4.2.13.Final
+io.netty:netty-codec:4.2.13.Final
+io.netty:netty-common:4.2.13.Final
+io.netty:netty-handler:4.2.13.Final
+io.netty:netty-resolver:4.2.13.Final
+io.netty:netty-transport-classes-epoll:4.2.13.Final
+io.netty:netty-transport-native-unix-common:4.2.13.Final
+io.netty:netty-transport:4.2.13.Final
+org.apache.httpcomponents:httpclient:4.5.13
+org.apache.httpcomponents:httpcore:4.4.16
+org.checkerframework:checker-qual:3.19.0
+org.reactivestreams:reactive-streams:1.0.4
+software.amazon.awssdk.crt:aws-crt:0.45.1
+software.amazon.awssdk:annotations:2.44.4
+software.amazon.awssdk:apache-client:2.44.4
+software.amazon.awssdk:arns:2.44.4
+software.amazon.awssdk:auth:2.44.4
+software.amazon.awssdk:aws-core:2.44.4
+software.amazon.awssdk:aws-json-protocol:2.44.4
+software.amazon.awssdk:aws-query-protocol:2.44.4
+software.amazon.awssdk:aws-xml-protocol:2.44.4
+software.amazon.awssdk:checksums-spi:2.44.4
+software.amazon.awssdk:checksums:2.44.4
+software.amazon.awssdk:cloudwatch-metric-publisher:2.44.4
+software.amazon.awssdk:cloudwatch:2.44.4
+software.amazon.awssdk:crt-core:2.44.4
+software.amazon.awssdk:dynamodb:2.44.4
+software.amazon.awssdk:endpoints-spi:2.44.4
+software.amazon.awssdk:glue:2.44.4
+software.amazon.awssdk:http-auth-aws-crt:2.44.4
+software.amazon.awssdk:http-auth-aws-eventstream:2.44.4
+software.amazon.awssdk:http-auth-aws:2.44.4
+software.amazon.awssdk:http-auth-spi:2.44.4
+software.amazon.awssdk:http-auth:2.44.4
+software.amazon.awssdk:http-client-spi:2.44.4
+software.amazon.awssdk:iam:2.44.4
+software.amazon.awssdk:identity-spi:2.44.4
+software.amazon.awssdk:json-utils:2.44.4
+software.amazon.awssdk:kms:2.44.4
+software.amazon.awssdk:lakeformation:2.44.4
+software.amazon.awssdk:metrics-spi:2.44.4
+software.amazon.awssdk:netty-nio-client:2.44.4
+software.amazon.awssdk:profiles:2.44.4
+software.amazon.awssdk:protocol-core:2.44.4
+software.amazon.awssdk:regions:2.44.4
+software.amazon.awssdk:retries-spi:2.44.4
+software.amazon.awssdk:retries:2.44.4
+software.amazon.awssdk:s3:2.44.4
+software.amazon.awssdk:s3control:2.44.4
+software.amazon.awssdk:sdk-core:2.44.4
+software.amazon.awssdk:smithy-rpcv2-protocol:2.44.4
+software.amazon.awssdk:sso:2.44.4
+software.amazon.awssdk:sts:2.44.4
+software.amazon.awssdk:third-party-jackson-core:2.44.4
+software.amazon.awssdk:third-party-jackson-dataformat-cbor:2.44.4
+software.amazon.awssdk:utils-lite:2.44.4
+software.amazon.awssdk:utils:2.44.4
+software.amazon.eventstream:eventstream:1.0.1
+software.amazon.s3.accessgrants:aws-s3-accessgrants-java-plugin:2.4.1
+software.amazon.s3.analyticsaccelerator:analyticsaccelerator-s3:1.3.1
diff --git a/aws/src/integration/java/org/apache/iceberg/aws/TestKeyManagementClient.java b/aws/src/integration/java/org/apache/iceberg/aws/TestKeyManagementClient.java
index 83bacf2601cd..ef84b23f8f27 100644
--- a/aws/src/integration/java/org/apache/iceberg/aws/TestKeyManagementClient.java
+++ b/aws/src/integration/java/org/apache/iceberg/aws/TestKeyManagementClient.java
@@ -22,6 +22,7 @@
import java.nio.ByteBuffer;
import java.util.Map;
+import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.encryption.KeyManagementClient;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.junit.jupiter.api.AfterAll;
@@ -31,6 +32,7 @@
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariables;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.NullSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,6 +40,7 @@
import software.amazon.awssdk.services.kms.model.CreateKeyRequest;
import software.amazon.awssdk.services.kms.model.CreateKeyResponse;
import software.amazon.awssdk.services.kms.model.DataKeySpec;
+import software.amazon.awssdk.services.kms.model.EncryptionAlgorithmSpec;
import software.amazon.awssdk.services.kms.model.KeySpec;
import software.amazon.awssdk.services.kms.model.ScheduleKeyDeletionRequest;
import software.amazon.awssdk.services.kms.model.ScheduleKeyDeletionResponse;
@@ -91,13 +94,42 @@ public void testKeyWrapping() {
try (AwsKeyManagementClient keyManagementClient = new AwsKeyManagementClient()) {
keyManagementClient.initialize(ImmutableMap.of());
- ByteBuffer key = ByteBuffer.wrap(new String("super-secret-table-master-key").getBytes());
+ ByteBuffer key = ByteBuffer.wrap("super-secret-table-master-key".getBytes());
ByteBuffer encryptedKey = keyManagementClient.wrapKey(key, keyId);
assertThat(keyManagementClient.unwrapKey(encryptedKey, keyId)).isEqualTo(key);
}
}
+ @ParameterizedTest
+ @MethodSource("org.apache.iceberg.TestHelpers#serializers")
+ public void testSerialization(
+ TestHelpers.RoundTripSerializer roundTripSerializer)
+ throws Exception {
+ try (AwsKeyManagementClient keyManagementClient = new AwsKeyManagementClient()) {
+ keyManagementClient.initialize(
+ ImmutableMap.of(
+ AwsProperties.KMS_ENCRYPTION_ALGORITHM_SPEC,
+ EncryptionAlgorithmSpec.RSAES_OAEP_SHA_256.toString(),
+ AwsProperties.KMS_DATA_KEY_SPEC,
+ DataKeySpec.AES_128.toString()));
+ assertThat(keyManagementClient.encryptionAlgorithmSpec())
+ .isEqualTo(EncryptionAlgorithmSpec.RSAES_OAEP_SHA_256);
+ assertThat(keyManagementClient.dataKeySpec()).isEqualTo(DataKeySpec.AES_128);
+
+ AwsKeyManagementClient result = roundTripSerializer.apply(keyManagementClient);
+
+ ByteBuffer key = ByteBuffer.wrap("super-secret-table-master-key".getBytes());
+ ByteBuffer encryptedKey = result.wrapKey(key, keyId);
+
+ assertThat(keyManagementClient.unwrapKey(encryptedKey, keyId)).isEqualTo(key);
+ assertThat(result.unwrapKey(encryptedKey, keyId)).isEqualTo(key);
+ assertThat(result.encryptionAlgorithmSpec())
+ .isEqualTo(EncryptionAlgorithmSpec.RSAES_OAEP_SHA_256);
+ assertThat(result.dataKeySpec()).isEqualTo(DataKeySpec.AES_128);
+ }
+ }
+
@ParameterizedTest
@NullSource
@EnumSource(
diff --git a/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java b/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java
index 65e37eba4cd3..b02537bf40b2 100644
--- a/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java
+++ b/aws/src/integration/java/org/apache/iceberg/aws/glue/GlueTestBase.java
@@ -65,6 +65,7 @@ public class GlueTestBase {
// iceberg
static GlueCatalog glueCatalog;
static GlueCatalog glueCatalogWithSkipNameValidation;
+ static GlueCatalog glueCatalogWithUniqueLocation;
static Schema schema =
new Schema(Types.NestedField.required(1, "c1", Types.StringType.get(), "c1"));
@@ -105,6 +106,16 @@ public static void beforeClass() {
GLUE,
null,
ImmutableMap.of());
+
+ glueCatalogWithUniqueLocation = new GlueCatalog();
+ glueCatalogWithUniqueLocation.initialize(
+ CATALOG_NAME,
+ TEST_BUCKET_PATH,
+ awsProperties,
+ s3FileIOProperties,
+ GLUE,
+ null,
+ true /* uniqTableLocation */);
}
@AfterAll
diff --git a/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogTable.java b/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogTable.java
index 2c9459c5e36c..cb015b79fb9b 100644
--- a/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogTable.java
+++ b/aws/src/integration/java/org/apache/iceberg/aws/glue/TestGlueCatalogTable.java
@@ -310,6 +310,22 @@ public void testRenameTable() {
assertThat(renamedTable.currentSnapshot()).isEqualTo(table.currentSnapshot());
}
+ @Test
+ public void testCreateTableInUniqueLocation() {
+ String namespace = createNamespace();
+ String tableName = createTable(namespace);
+ String newTableName = tableName + "_renamed";
+
+ glueCatalogWithUniqueLocation.renameTable(
+ TableIdentifier.of(namespace, tableName), TableIdentifier.of(namespace, newTableName));
+ Table renamedTable =
+ glueCatalogWithUniqueLocation.loadTable(TableIdentifier.of(namespace, newTableName));
+ createTable(namespace, tableName);
+ Table table = glueCatalogWithUniqueLocation.loadTable(TableIdentifier.of(namespace, tableName));
+
+ assertThat(renamedTable.location()).isNotEqualTo(table.location());
+ }
+
@Test
public void testRenameTableFailsToCreateNewTable() {
String namespace = createNamespace();
@@ -743,7 +759,8 @@ public void testTableLevelS3Tags() {
new AwsProperties(properties),
new S3FileIOProperties(properties),
GLUE,
- null);
+ null,
+ false /* uniqTableLocation */);
String namespace = createNamespace();
String tableName = getRandomName();
createTable(namespace, tableName);
diff --git a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3FileIO.java b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3FileIO.java
index ebee07e53e4c..e2fe6db0a494 100644
--- a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3FileIO.java
+++ b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3FileIO.java
@@ -986,4 +986,60 @@ private void createBucket(String bucketName) {
// do nothing
}
}
+
+ @Test
+ public void setCredentialsRefreshesClients() {
+ StorageCredential initialCredential =
+ StorageCredential.create(
+ "s3://custom-uri",
+ ImmutableMap.of(
+ "s3.access-key-id",
+ "initialKeyId",
+ "s3.secret-access-key",
+ "initialSecretKey",
+ "s3.session-token",
+ "initialSessionToken"));
+
+ S3FileIO fileIO = new S3FileIO();
+ fileIO.setCredentials(ImmutableList.of(initialCredential));
+ fileIO.initialize(ImmutableMap.of(AwsClientProperties.CLIENT_REGION, "us-east-1"));
+
+ S3Client initialClient = fileIO.client("s3://custom-uri/table1");
+ assertThat(initialClient.serviceClientConfiguration())
+ .extracting(AwsServiceClientConfiguration::credentialsProvider)
+ .extracting(IdentityProvider::resolveIdentity)
+ .satisfies(
+ future -> {
+ AwsSessionCredentialsIdentity identity = (AwsSessionCredentialsIdentity) future.get();
+ assertThat(identity.accessKeyId()).isEqualTo("initialKeyId");
+ assertThat(identity.secretAccessKey()).isEqualTo("initialSecretKey");
+ assertThat(identity.sessionToken()).isEqualTo("initialSessionToken");
+ });
+
+ StorageCredential refreshedCredential =
+ StorageCredential.create(
+ "s3://custom-uri",
+ ImmutableMap.of(
+ "s3.access-key-id",
+ "refreshedKeyId",
+ "s3.secret-access-key",
+ "refreshedSecretKey",
+ "s3.session-token",
+ "refreshedSessionToken"));
+
+ fileIO.setCredentials(ImmutableList.of(refreshedCredential));
+
+ S3Client refreshedClient = fileIO.client("s3://custom-uri/table1");
+ assertThat(refreshedClient).isNotSameAs(initialClient);
+ assertThat(refreshedClient.serviceClientConfiguration())
+ .extracting(AwsServiceClientConfiguration::credentialsProvider)
+ .extracting(IdentityProvider::resolveIdentity)
+ .satisfies(
+ x -> {
+ AwsSessionCredentialsIdentity identity = (AwsSessionCredentialsIdentity) x.get();
+ assertThat(identity.accessKeyId()).isEqualTo("refreshedKeyId");
+ assertThat(identity.secretAccessKey()).isEqualTo("refreshedSecretKey");
+ assertThat(identity.sessionToken()).isEqualTo("refreshedSessionToken");
+ });
+ }
}
diff --git a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java
index cbe3051a6711..746015098a40 100644
--- a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java
+++ b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java
@@ -21,12 +21,14 @@
import static org.assertj.core.api.Assertions.assertThat;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Random;
import java.util.UUID;
import java.util.function.Supplier;
import java.util.stream.IntStream;
import org.apache.iceberg.aws.AwsClientFactories;
import org.apache.iceberg.aws.AwsIntegTestUtil;
+import org.apache.iceberg.io.IOUtil;
import org.apache.iceberg.io.PositionOutputStream;
import org.apache.iceberg.io.SeekableInputStream;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
@@ -36,6 +38,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariables;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import software.amazon.awssdk.services.s3.S3Client;
/** Long-running tests to ensure multipart upload logic is resilient */
@@ -141,6 +145,35 @@ public void testParallelUpload() throws IOException {
}
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testMultipartUploadWithChunkedEncoding(boolean chunkedEncodingEnabled)
+ throws IOException {
+ // Create a new S3FileIO with specified chunked encoding setting
+ try (S3FileIO testIo = new S3FileIO(() -> s3)) {
+ testIo.initialize(
+ ImmutableMap.of(
+ S3FileIOProperties.MULTIPART_SIZE,
+ Integer.toString(S3FileIOProperties.MULTIPART_SIZE_MIN),
+ S3FileIOProperties.CHECKSUM_ENABLED,
+ "true",
+ S3FileIOProperties.CHUNKED_ENCODING_ENABLED,
+ Boolean.toString(chunkedEncodingEnabled)));
+
+ int parts = 10;
+ long partSize = S3FileIOProperties.MULTIPART_SIZE_MIN;
+ String suffix = chunkedEncodingEnabled ? "-chunked-enabled" : "-chunked-disabled";
+
+ String intObjectUri = objectUri + suffix + "-int";
+ writeDistinctPartsWithInts(testIo, intObjectUri, parts, partSize);
+ verifyDistinctPartsWithInts(testIo, intObjectUri, parts, partSize);
+
+ String bytesObjectUri = objectUri + suffix + "-bytes";
+ writeDistinctPartsWithBytes(testIo, bytesObjectUri, parts, partSize);
+ verifyDistinctPartsWithBytes(testIo, bytesObjectUri, parts, partSize);
+ }
+ }
+
private void writeInts(String fileUri, int parts, Supplier writer) {
writeInts(fileUri, parts, S3FileIOProperties.MULTIPART_SIZE_MIN, writer);
}
@@ -177,4 +210,61 @@ private void writeBytes(String fileUri, int parts, Supplier writer) {
throw new RuntimeException(e);
}
}
+
+ private void writeDistinctPartsWithInts(S3FileIO fileIO, String fileUri, int parts, long partSize)
+ throws IOException {
+ try (PositionOutputStream outputStream = fileIO.newOutputFile(fileUri).create()) {
+ for (int part = 0; part < parts; part++) {
+ int partByte = part + 1;
+ for (long j = 0; j < partSize; j++) {
+ outputStream.write(partByte);
+ }
+ }
+ }
+
+ assertThat(fileIO.newInputFile(fileUri).getLength()).isEqualTo(parts * partSize);
+ }
+
+ private void verifyDistinctPartsWithInts(
+ S3FileIO fileIO, String fileUri, int parts, long partSize) throws IOException {
+ try (SeekableInputStream inputStream = fileIO.newInputFile(fileUri).newStream()) {
+ byte[] readBuffer = new byte[(int) partSize];
+ for (int part = 0; part < parts; part++) {
+ byte expectedByte = (byte) (part + 1);
+ IOUtil.readFully(inputStream, readBuffer, 0, (int) partSize);
+ for (int i = 0; i < (int) partSize; i++) {
+ assertThat(readBuffer[i]).as("part %d, offset %d", part, i).isEqualTo(expectedByte);
+ }
+ }
+ assertThat(inputStream.read()).as("expected end of stream").isEqualTo(-1);
+ }
+ }
+
+ private void writeDistinctPartsWithBytes(
+ S3FileIO fileIO, String fileUri, int parts, long partSize) throws IOException {
+ try (PositionOutputStream outputStream = fileIO.newOutputFile(fileUri).create()) {
+ for (int part = 0; part < parts; part++) {
+ byte[] partBytes = new byte[(int) partSize];
+ Arrays.fill(partBytes, (byte) (part + 1));
+ outputStream.write(partBytes);
+ }
+ }
+
+ assertThat(fileIO.newInputFile(fileUri).getLength()).isEqualTo(parts * partSize);
+ }
+
+ private void verifyDistinctPartsWithBytes(
+ S3FileIO fileIO, String fileUri, int parts, long partSize) throws IOException {
+ try (SeekableInputStream inputStream = fileIO.newInputFile(fileUri).newStream()) {
+ byte[] readBuffer = new byte[(int) partSize];
+ for (int part = 0; part < parts; part++) {
+ byte expectedByte = (byte) (part + 1);
+ IOUtil.readFully(inputStream, readBuffer, 0, (int) partSize);
+ for (int i = 0; i < (int) partSize; i++) {
+ assertThat(readBuffer[i]).as("part %d, offset %d", part, i).isEqualTo(expectedByte);
+ }
+ }
+ assertThat(inputStream.read()).as("expected end of stream").isEqualTo(-1);
+ }
+ }
}
diff --git a/aws/src/integration/java/org/apache/iceberg/aws/s3/signer/S3SignerServlet.java b/aws/src/integration/java/org/apache/iceberg/aws/s3/signer/S3SignerServlet.java
index 038d76b03e4b..5d334eafa582 100644
--- a/aws/src/integration/java/org/apache/iceberg/aws/s3/signer/S3SignerServlet.java
+++ b/aws/src/integration/java/org/apache/iceberg/aws/s3/signer/S3SignerServlet.java
@@ -18,17 +18,6 @@
*/
package org.apache.iceberg.aws.s3.signer;
-import static java.lang.String.format;
-import static org.apache.iceberg.rest.RESTCatalogAdapter.castRequest;
-import static org.apache.iceberg.rest.RESTCatalogAdapter.castResponse;
-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import jakarta.servlet.http.HttpServlet;
-import jakarta.servlet.http.HttpServletRequest;
-import jakarta.servlet.http.HttpServletResponse;
-import java.io.InputStreamReader;
-import java.io.Reader;
import java.time.Clock;
import java.time.Instant;
import java.time.ZoneId;
@@ -37,23 +26,15 @@
import java.util.Locale;
import java.util.Map;
import java.util.Set;
-import java.util.function.Predicate;
import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import org.apache.hc.core5.http.ContentType;
-import org.apache.hc.core5.http.HttpHeaders;
-import org.apache.iceberg.exceptions.RESTException;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
-import org.apache.iceberg.relocated.com.google.common.io.CharStreams;
-import org.apache.iceberg.rest.RESTUtil;
-import org.apache.iceberg.rest.ResourcePaths;
-import org.apache.iceberg.rest.responses.ErrorResponse;
-import org.apache.iceberg.rest.responses.OAuthTokenResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.iceberg.rest.HttpMethod;
+import org.apache.iceberg.rest.RemoteSignerServlet;
+import org.apache.iceberg.rest.requests.RemoteSignRequest;
+import org.apache.iceberg.rest.responses.ImmutableRemoteSignResponse;
+import org.apache.iceberg.rest.responses.RemoteSignResponse;
import software.amazon.awssdk.auth.signer.AwsS3V4Signer;
import software.amazon.awssdk.auth.signer.params.AwsS3V4SignerParams;
import software.amazon.awssdk.http.SdkHttpFullRequest;
@@ -65,113 +46,37 @@
* {@link S3SignerServlet} provides a simple servlet implementation to emulate the server-side
* behavior of signing S3 requests and handling OAuth.
*/
-public class S3SignerServlet extends HttpServlet {
-
- private static final Logger LOG = LoggerFactory.getLogger(S3SignerServlet.class);
+public class S3SignerServlet extends RemoteSignerServlet {
static final Clock SIGNING_CLOCK = Clock.fixed(Instant.now(), ZoneId.of("UTC"));
static final Set UNSIGNED_HEADERS =
Sets.newHashSet(
Arrays.asList("range", "x-amz-date", "amz-sdk-invocation-id", "amz-sdk-retry"));
- private static final String POST = "POST";
-
- private static final Set CACHEABLE_METHODS =
- Stream.of(SdkHttpMethod.GET, SdkHttpMethod.HEAD).collect(Collectors.toSet());
-
- private final Map responseHeaders =
- ImmutableMap.of(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.getMimeType());
- private final ObjectMapper mapper;
-
- private List s3SignRequestValidators = Lists.newArrayList();
-
- /**
- * SignRequestValidator is a wrapper class used for validating the contents of the S3SignRequest
- * and thus verifying the behavior of the client during testing.
- */
- public static class SignRequestValidator {
- private final Predicate requestMatcher;
- private final Predicate requestExpectation;
- private final String assertMessage;
-
- public SignRequestValidator(
- Predicate requestExpectation,
- Predicate requestMatcher,
- String assertMessage) {
- this.requestExpectation = requestExpectation;
- this.requestMatcher = requestMatcher;
- this.assertMessage = assertMessage;
- }
-
- void validateRequest(S3SignRequest request) {
- if (requestMatcher.test(request)) {
- assertThat(requestExpectation.test(request)).as(assertMessage).isTrue();
- }
- }
- }
-
- public S3SignerServlet(ObjectMapper mapper) {
- this.mapper = mapper;
- }
-
- public S3SignerServlet(ObjectMapper mapper, List s3SignRequestValidators) {
- this.mapper = mapper;
- this.s3SignRequestValidators = s3SignRequestValidators;
- }
-
- @Override
- protected void doGet(HttpServletRequest request, HttpServletResponse response) {
- execute(request, response);
- }
- @Override
- protected void doHead(HttpServletRequest request, HttpServletResponse response) {
- execute(request, response);
- }
+ /** A fake remote signing endpoint for testing purposes. */
+ static final String S3_SIGNER_ENDPOINT = "v1/namespaces/ns1/tables/t1/sign";
- @Override
- protected void doPost(HttpServletRequest request, HttpServletResponse response) {
- execute(request, response);
+ public S3SignerServlet() {
+ super(S3_SIGNER_ENDPOINT);
}
@Override
- protected void doDelete(HttpServletRequest request, HttpServletResponse response) {
- execute(request, response);
- }
-
- private OAuthTokenResponse handleOAuth(Map requestMap) {
- String grantType = requestMap.get("grant_type");
- switch (grantType) {
- case "client_credentials":
- return castResponse(
- OAuthTokenResponse.class,
- OAuthTokenResponse.builder()
- .withToken("client-credentials-token:sub=" + requestMap.get("client_id"))
- .withIssuedTokenType("urn:ietf:params:oauth:token-type:access_token")
- .withTokenType("Bearer")
- .setExpirationInSeconds(10000)
- .build());
-
- case "urn:ietf:params:oauth:grant-type:token-exchange":
- String actor = requestMap.get("actor_token");
- String token =
- String.format(
- "token-exchange-token:sub=%s%s",
- requestMap.get("subject_token"), actor != null ? ",act=" + actor : "");
- return castResponse(
- OAuthTokenResponse.class,
- OAuthTokenResponse.builder()
- .withToken(token)
- .withIssuedTokenType("urn:ietf:params:oauth:token-type:access_token")
- .withTokenType("Bearer")
- .setExpirationInSeconds(10000)
- .build());
-
- default:
- throw new UnsupportedOperationException("Unsupported grant_type: " + grantType);
+ protected void validateSignRequest(RemoteSignRequest request) {
+ Preconditions.checkArgument(
+ request.provider() == null || "s3".equalsIgnoreCase(request.provider()),
+ "Unsupported provider: %s",
+ request.provider());
+ if (HttpMethod.POST.name().equalsIgnoreCase(request.method())
+ && request.uri().getQuery().contains("delete")) {
+ String body = request.body();
+ Preconditions.checkArgument(
+ body != null && !body.isEmpty(),
+ "Sign request for delete objects should have a request body");
}
}
- private S3SignResponse signRequest(S3SignRequest request) {
+ @Override
+ protected RemoteSignResponse signRequest(RemoteSignRequest request) {
AwsS3V4SignerParams signingParams =
AwsS3V4SignerParams.builder()
.awsCredentials(TestS3RestSigner.CREDENTIALS_PROVIDER.resolveCredentials())
@@ -207,59 +112,6 @@ private S3SignResponse signRequest(S3SignRequest request) {
Map> headers = Maps.newHashMap(sign.headers());
headers.putAll(unsignedHeaders);
- return ImmutableS3SignResponse.builder().uri(request.uri()).headers(headers).build();
- }
-
- protected void execute(HttpServletRequest request, HttpServletResponse response) {
- response.setStatus(HttpServletResponse.SC_OK);
- responseHeaders.forEach(response::setHeader);
-
- String path = request.getRequestURI().substring(1);
- Object requestBody;
- try {
- // we only need to handle oauth tokens & s3 sign request routes here as those are the only
- // requests that are being done by the S3V4RestSignerClient
- if (POST.equals(request.getMethod())
- && S3V4RestSignerClient.S3_SIGNER_DEFAULT_ENDPOINT.equals(path)) {
- S3SignRequest s3SignRequest =
- castRequest(
- S3SignRequest.class, mapper.readValue(request.getReader(), S3SignRequest.class));
- s3SignRequestValidators.forEach(validator -> validator.validateRequest(s3SignRequest));
- S3SignResponse s3SignResponse = signRequest(s3SignRequest);
- if (CACHEABLE_METHODS.contains(SdkHttpMethod.fromValue(s3SignRequest.method()))) {
- // tell the client this can be cached
- response.setHeader(
- S3V4RestSignerClient.CACHE_CONTROL, S3V4RestSignerClient.CACHE_CONTROL_PRIVATE);
- } else {
- response.setHeader(
- S3V4RestSignerClient.CACHE_CONTROL, S3V4RestSignerClient.CACHE_CONTROL_NO_CACHE);
- }
-
- mapper.writeValue(response.getWriter(), s3SignResponse);
- } else if (POST.equals(request.getMethod()) && ResourcePaths.tokens().equals(path)) {
- try (Reader reader = new InputStreamReader(request.getInputStream())) {
- requestBody = RESTUtil.decodeFormData(CharStreams.toString(reader));
- }
-
- OAuthTokenResponse oAuthTokenResponse =
- handleOAuth((Map) castRequest(Map.class, requestBody));
- mapper.writeValue(response.getWriter(), oAuthTokenResponse);
- } else {
- response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
- mapper.writeValue(
- response.getWriter(),
- ErrorResponse.builder()
- .responseCode(400)
- .withType("BadRequestException")
- .withMessage(format("No route for request: %s %s", request.getMethod(), path))
- .build());
- }
- } catch (RESTException e) {
- LOG.error("Error processing REST request", e);
- response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
- } catch (Exception e) {
- LOG.error("Unexpected exception when processing REST request", e);
- response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
- }
+ return ImmutableRemoteSignResponse.builder().uri(request.uri()).headers(headers).build();
}
}
diff --git a/aws/src/integration/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java b/aws/src/integration/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java
index b51d97cc611a..4e5ed3d91870 100644
--- a/aws/src/integration/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java
+++ b/aws/src/integration/java/org/apache/iceberg/aws/s3/signer/TestS3RestSigner.java
@@ -33,14 +33,15 @@
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import org.apache.iceberg.aws.s3.MinioUtil;
-import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.rest.RESTCatalogProperties;
import org.apache.iceberg.rest.auth.OAuth2Properties;
import org.apache.iceberg.util.ThreadPools;
+import org.eclipse.jetty.compression.gzip.GzipCompression;
+import org.eclipse.jetty.compression.server.CompressionHandler;
+import org.eclipse.jetty.ee10.servlet.ServletContextHandler;
+import org.eclipse.jetty.ee10.servlet.ServletHolder;
import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.handler.gzip.GzipHandler;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
@@ -107,8 +108,10 @@ public static void beforeClass() throws Exception {
ImmutableS3V4RestSignerClient.builder()
.properties(
ImmutableMap.of(
- S3V4RestSignerClient.S3_SIGNER_URI,
+ RESTCatalogProperties.SIGNER_URI,
httpServer.getURI().toString(),
+ RESTCatalogProperties.SIGNER_ENDPOINT,
+ S3SignerServlet.S3_SIGNER_ENDPOINT,
OAuth2Properties.CREDENTIAL,
"catalog:12345"))
.build(),
@@ -182,19 +185,13 @@ public void before() throws Exception {
}
private static Server initHttpServer() throws Exception {
- S3SignerServlet.SignRequestValidator deleteObjectsWithBody =
- new S3SignerServlet.SignRequestValidator(
- (s3SignRequest) ->
- "post".equalsIgnoreCase(s3SignRequest.method())
- && s3SignRequest.uri().getQuery().contains("delete"),
- (s3SignRequest) -> s3SignRequest.body() != null && !s3SignRequest.body().isEmpty(),
- "Sign request for delete objects should have a request body");
- S3SignerServlet servlet =
- new S3SignerServlet(S3ObjectMapper.mapper(), ImmutableList.of(deleteObjectsWithBody));
+ S3SignerServlet servlet = new S3SignerServlet();
ServletContextHandler servletContext =
new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
servletContext.addServlet(new ServletHolder(servlet), "/*");
- servletContext.setHandler(new GzipHandler());
+ CompressionHandler compressionHandler = new CompressionHandler();
+ compressionHandler.putCompression(new GzipCompression());
+ servletContext.insertHandler(compressionHandler);
Server server = new Server(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0));
server.setHandler(servletContext);
diff --git a/aws/src/main/java/org/apache/iceberg/aws/ApacheHttpClientConfigurations.java b/aws/src/main/java/org/apache/iceberg/aws/ApacheHttpClientConfigurations.java
index 3445928d1551..30065c8db510 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/ApacheHttpClientConfigurations.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/ApacheHttpClientConfigurations.java
@@ -41,6 +41,8 @@ class ApacheHttpClientConfigurations extends BaseHttpClientConfigurations {
private Boolean tcpKeepAliveEnabled;
private Boolean useIdleConnectionReaperEnabled;
private String proxyEndpoint;
+ private Boolean proxyUseSystemPropertyValues;
+ private Boolean proxyUseEnvironmentVariableValues;
private ApacheHttpClientConfigurations() {}
@@ -82,6 +84,12 @@ private void initialize(Map httpClientProperties) {
this.proxyEndpoint =
PropertyUtil.propertyAsString(
httpClientProperties, HttpClientProperties.PROXY_ENDPOINT, null);
+ this.proxyUseSystemPropertyValues =
+ PropertyUtil.propertyAsNullableBoolean(
+ httpClientProperties, HttpClientProperties.PROXY_USE_SYSTEM_PROPERTY_VALUES);
+ this.proxyUseEnvironmentVariableValues =
+ PropertyUtil.propertyAsNullableBoolean(
+ httpClientProperties, HttpClientProperties.PROXY_USE_ENVIRONMENT_VARIABLE_VALUES);
}
@VisibleForTesting
@@ -113,9 +121,26 @@ void configureApacheHttpClientBuilder(ApacheHttpClient.Builder apacheHttpClientB
if (useIdleConnectionReaperEnabled != null) {
apacheHttpClientBuilder.useIdleConnectionReaper(useIdleConnectionReaperEnabled);
}
- if (proxyEndpoint != null) {
- apacheHttpClientBuilder.proxyConfiguration(
- ProxyConfiguration.builder().endpoint(URI.create(proxyEndpoint)).build());
+ configureProxy(apacheHttpClientBuilder);
+ }
+
+ private void configureProxy(ApacheHttpClient.Builder apacheHttpClientBuilder) {
+ if (proxyEndpoint != null
+ || proxyUseSystemPropertyValues != null
+ || proxyUseEnvironmentVariableValues != null) {
+ ProxyConfiguration.Builder proxyBuilder = ProxyConfiguration.builder();
+
+ if (proxyEndpoint != null) {
+ proxyBuilder.endpoint(URI.create(proxyEndpoint));
+ }
+ if (proxyUseSystemPropertyValues != null) {
+ proxyBuilder.useSystemPropertyValues(proxyUseSystemPropertyValues);
+ }
+ if (proxyUseEnvironmentVariableValues != null) {
+ proxyBuilder.useEnvironmentVariableValues(proxyUseEnvironmentVariableValues);
+ }
+
+ apacheHttpClientBuilder.proxyConfiguration(proxyBuilder.build());
}
}
@@ -138,6 +163,8 @@ protected String generateHttpClientCacheKey() {
keyComponents.put("tcpKeepAliveEnabled", tcpKeepAliveEnabled);
keyComponents.put("useIdleConnectionReaperEnabled", useIdleConnectionReaperEnabled);
keyComponents.put("proxyEndpoint", proxyEndpoint);
+ keyComponents.put("proxyUseSystemPropertyValues", proxyUseSystemPropertyValues);
+ keyComponents.put("proxyUseEnvironmentVariableValues", proxyUseEnvironmentVariableValues);
return keyComponents.entrySet().stream()
.map(entry -> entry.getKey() + "=" + Objects.toString(entry.getValue(), "null"))
diff --git a/aws/src/main/java/org/apache/iceberg/aws/AssumeRoleAwsClientFactory.java b/aws/src/main/java/org/apache/iceberg/aws/AssumeRoleAwsClientFactory.java
index 59a4d8d3ac38..dd955d579761 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/AssumeRoleAwsClientFactory.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/AssumeRoleAwsClientFactory.java
@@ -81,6 +81,7 @@ public GlueClient glue() {
return GlueClient.builder()
.applyMutation(this::applyAssumeRoleConfigurations)
.applyMutation(httpClientProperties::applyHttpClientConfigurations)
+ .applyMutation(awsClientProperties::applyRetryConfigurations)
.build();
}
@@ -99,6 +100,7 @@ public DynamoDbClient dynamo() {
.applyMutation(this::applyAssumeRoleConfigurations)
.applyMutation(httpClientProperties::applyHttpClientConfigurations)
.applyMutation(awsProperties::applyDynamoDbEndpointConfigurations)
+ .applyMutation(awsClientProperties::applyRetryConfigurations)
.build();
}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java b/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java
index 0ee4bf26e68e..e6eb8d28d01a 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/AwsClientFactories.java
@@ -148,6 +148,7 @@ public GlueClient glue() {
.applyMutation(httpClientProperties::applyHttpClientConfigurations)
.applyMutation(awsProperties::applyGlueEndpointConfigurations)
.applyMutation(awsClientProperties::applyClientCredentialConfigurations)
+ .applyMutation(awsClientProperties::applyRetryConfigurations)
.build();
}
@@ -169,6 +170,7 @@ public DynamoDbClient dynamo() {
.applyMutation(httpClientProperties::applyHttpClientConfigurations)
.applyMutation(awsClientProperties::applyClientCredentialConfigurations)
.applyMutation(awsProperties::applyDynamoDbEndpointConfigurations)
+ .applyMutation(awsClientProperties::applyRetryConfigurations)
.build();
}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/AwsClientProperties.java b/aws/src/main/java/org/apache/iceberg/aws/AwsClientProperties.java
index cf73e80f44c1..c5850d0c79c1 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/AwsClientProperties.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/AwsClientProperties.java
@@ -20,6 +20,7 @@
import java.io.Serializable;
import java.util.Map;
+import java.util.function.Predicate;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.aws.s3.VendedCredentialsProvider;
import org.apache.iceberg.common.DynClasses;
@@ -28,7 +29,6 @@
import org.apache.iceberg.relocated.com.google.common.base.Strings;
import org.apache.iceberg.rest.RESTUtil;
import org.apache.iceberg.util.PropertyUtil;
-import org.apache.iceberg.util.SerializableMap;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
@@ -97,7 +97,6 @@ public class AwsClientProperties implements Serializable {
private final String refreshCredentialsEndpoint;
private final boolean refreshCredentialsEnabled;
private final boolean legacyMd5pluginEnabled;
- private final Map allProperties;
public AwsClientProperties() {
this.clientRegion = null;
@@ -106,15 +105,18 @@ public AwsClientProperties() {
this.refreshCredentialsEndpoint = null;
this.refreshCredentialsEnabled = true;
this.legacyMd5pluginEnabled = false;
- this.allProperties = null;
}
public AwsClientProperties(Map properties) {
- this.allProperties = SerializableMap.copyOf(properties);
this.clientRegion = properties.get(CLIENT_REGION);
this.clientCredentialsProvider = properties.get(CLIENT_CREDENTIALS_PROVIDER);
+ // Retain all non-prefixed properties and override with prefixed properties
this.clientCredentialsProviderProperties =
- PropertyUtil.propertiesWithPrefix(properties, CLIENT_CREDENTIAL_PROVIDER_PREFIX);
+ PropertyUtil.mergeProperties(
+ PropertyUtil.filterProperties(
+ properties,
+ Predicate.not(property -> property.startsWith(CLIENT_CREDENTIAL_PROVIDER_PREFIX))),
+ PropertyUtil.propertiesWithPrefix(properties, CLIENT_CREDENTIAL_PROVIDER_PREFIX));
this.refreshCredentialsEndpoint =
RESTUtil.resolveEndpoint(
properties.get(CatalogProperties.URI), properties.get(REFRESH_CREDENTIALS_ENDPOINT));
@@ -210,8 +212,13 @@ public void applyClientCredentialConfigurati
@SuppressWarnings("checkstyle:HiddenField")
public AwsCredentialsProvider credentialsProvider(
String accessKeyId, String secretAccessKey, String sessionToken) {
+ if (!Strings.isNullOrEmpty(this.clientCredentialsProvider)) {
+ clientCredentialsProviderProperties.put(
+ VendedCredentialsProvider.URI, refreshCredentialsEndpoint);
+ return credentialsProvider(this.clientCredentialsProvider);
+ }
+
if (refreshCredentialsEnabled && !Strings.isNullOrEmpty(refreshCredentialsEndpoint)) {
- clientCredentialsProviderProperties.putAll(allProperties);
clientCredentialsProviderProperties.put(
VendedCredentialsProvider.URI, refreshCredentialsEndpoint);
return credentialsProvider(VendedCredentialsProvider.class.getName());
@@ -227,10 +234,6 @@ public AwsCredentialsProvider credentialsProvider(
}
}
- if (!Strings.isNullOrEmpty(this.clientCredentialsProvider)) {
- return credentialsProvider(this.clientCredentialsProvider);
- }
-
// Create a new credential provider for each client
return DefaultCredentialsProvider.builder().build();
}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/AwsKeyManagementClient.java b/aws/src/main/java/org/apache/iceberg/aws/AwsKeyManagementClient.java
index 6d2671f4e26d..3b1c13ebe36f 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/AwsKeyManagementClient.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/AwsKeyManagementClient.java
@@ -20,7 +20,9 @@
import java.nio.ByteBuffer;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.iceberg.encryption.KeyManagementClient;
+import org.apache.iceberg.util.SerializableMap;
import software.amazon.awssdk.core.SdkBytes;
import software.amazon.awssdk.services.kms.KmsClient;
import software.amazon.awssdk.services.kms.model.DataKeySpec;
@@ -39,14 +41,17 @@
*/
public class AwsKeyManagementClient implements KeyManagementClient {
- private KmsClient kmsClient;
+ private final AtomicBoolean isResourceClosed = new AtomicBoolean(false);
+
+ private Map allProperties;
private EncryptionAlgorithmSpec encryptionAlgorithmSpec;
private DataKeySpec dataKeySpec;
+ private transient volatile KmsClient kmsClient;
+
@Override
public void initialize(Map properties) {
- AwsClientFactory clientFactory = AwsClientFactories.from(properties);
- this.kmsClient = clientFactory.kms();
+ this.allProperties = SerializableMap.copyOf(properties);
AwsProperties awsProperties = new AwsProperties(properties);
this.encryptionAlgorithmSpec = awsProperties.kmsEncryptionAlgorithmSpec();
@@ -62,7 +67,7 @@ public ByteBuffer wrapKey(ByteBuffer key, String wrappingKeyId) {
.plaintext(SdkBytes.fromByteBuffer(key))
.build();
- EncryptResponse result = kmsClient.encrypt(request);
+ EncryptResponse result = kmsClient().encrypt(request);
return result.ciphertextBlob().asByteBuffer();
}
@@ -76,11 +81,9 @@ public KeyGenerationResult generateKey(String wrappingKeyId) {
GenerateDataKeyRequest request =
GenerateDataKeyRequest.builder().keyId(wrappingKeyId).keySpec(dataKeySpec).build();
- GenerateDataKeyResponse response = kmsClient.generateDataKey(request);
- KeyGenerationResult result =
- new KeyGenerationResult(
- response.plaintext().asByteBuffer(), response.ciphertextBlob().asByteBuffer());
- return result;
+ GenerateDataKeyResponse response = kmsClient().generateDataKey(request);
+ return new KeyGenerationResult(
+ response.plaintext().asByteBuffer(), response.ciphertextBlob().asByteBuffer());
}
@Override
@@ -92,14 +95,36 @@ public ByteBuffer unwrapKey(ByteBuffer wrappedKey, String wrappingKeyId) {
.ciphertextBlob(SdkBytes.fromByteBuffer(wrappedKey))
.build();
- DecryptResponse result = kmsClient.decrypt(request);
+ DecryptResponse result = kmsClient().decrypt(request);
return result.plaintext().asByteBuffer();
}
@Override
public void close() {
- if (kmsClient != null) {
- kmsClient.close();
+ if (isResourceClosed.compareAndSet(false, true)) {
+ if (kmsClient != null) {
+ kmsClient.close();
+ }
+ }
+ }
+
+ EncryptionAlgorithmSpec encryptionAlgorithmSpec() {
+ return encryptionAlgorithmSpec;
+ }
+
+ DataKeySpec dataKeySpec() {
+ return dataKeySpec;
+ }
+
+ private KmsClient kmsClient() {
+ if (kmsClient == null) {
+ synchronized (this) {
+ if (kmsClient == null) {
+ AwsClientFactory clientFactory = AwsClientFactories.from(allProperties);
+ kmsClient = clientFactory.kms();
+ }
+ }
}
+ return kmsClient;
}
}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/HttpClientProperties.java b/aws/src/main/java/org/apache/iceberg/aws/HttpClientProperties.java
index 438ae5bb0431..870d8e23651c 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/HttpClientProperties.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/HttpClientProperties.java
@@ -61,6 +61,30 @@ public class HttpClientProperties implements Serializable {
*/
public static final String PROXY_ENDPOINT = "http-client.proxy-endpoint";
+ /**
+ * Used to enable reading proxy configuration from Java system properties (http.proxyHost,
+ * http.proxyPort, http.nonProxyHosts, etc.). Default is true.
+ *
+ * For more details, see
+ * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/urlconnection/ProxyConfiguration.html
+ * and
+ * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ProxyConfiguration.html
+ */
+ public static final String PROXY_USE_SYSTEM_PROPERTY_VALUES =
+ "http-client.proxy-use-system-property-values";
+
+ /**
+ * Used to enable reading proxy configuration from environment variables (HTTP_PROXY, HTTPS_PROXY,
+ * NO_PROXY, etc.). Default is true.
+ *
+ *
For more details, see
+ * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/urlconnection/ProxyConfiguration.html
+ * and
+ * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/http/apache/ProxyConfiguration.html
+ */
+ public static final String PROXY_USE_ENVIRONMENT_VARIABLE_VALUES =
+ "http-client.proxy-use-environment-variable-values";
+
/**
* Used to configure the connection timeout in milliseconds for {@link
* software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient.Builder}. This flag only
diff --git a/aws/src/main/java/org/apache/iceberg/aws/RESTSigV4AuthManager.java b/aws/src/main/java/org/apache/iceberg/aws/RESTSigV4AuthManager.java
index e975c9588eaa..60a944b3d6bd 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/RESTSigV4AuthManager.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/RESTSigV4AuthManager.java
@@ -36,7 +36,9 @@
*/
public class RESTSigV4AuthManager implements AuthManager {
+ @SuppressWarnings("deprecation")
private final Aws4Signer signer = Aws4Signer.create();
+
private final AuthManager delegate;
private Map catalogProperties = Map.of();
diff --git a/aws/src/main/java/org/apache/iceberg/aws/RESTSigV4AuthSession.java b/aws/src/main/java/org/apache/iceberg/aws/RESTSigV4AuthSession.java
index caee74114fbb..48281841be37 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/RESTSigV4AuthSession.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/RESTSigV4AuthSession.java
@@ -18,12 +18,15 @@
*/
package org.apache.iceberg.aws;
+import java.io.IOException;
+import java.io.UncheckedIOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
+import org.apache.iceberg.io.CloseableGroup;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.rest.HTTPHeaders;
import org.apache.iceberg.rest.HTTPHeaders.HTTPHeader;
@@ -57,20 +60,30 @@ public class RESTSigV4AuthSession implements AuthSession {
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
static final String RELOCATED_HEADER_PREFIX = "Original-";
+ @SuppressWarnings("deprecation")
private final Aws4Signer signer;
+
private final AuthSession delegate;
private final Region signingRegion;
private final String signingName;
private final AwsCredentialsProvider credentialsProvider;
+ private final CloseableGroup closeableGroup;
+ @SuppressWarnings("deprecation")
public RESTSigV4AuthSession(
Aws4Signer aws4Signer, AuthSession delegateAuthSession, AwsProperties awsProperties) {
+ this.closeableGroup = new CloseableGroup();
+ this.closeableGroup.setSuppressCloseFailure(true);
this.signer = Preconditions.checkNotNull(aws4Signer, "Invalid signer: null");
this.delegate = Preconditions.checkNotNull(delegateAuthSession, "Invalid delegate: null");
+ this.closeableGroup.addCloseable(this.delegate);
Preconditions.checkNotNull(awsProperties, "Invalid AWS properties: null");
this.signingRegion = awsProperties.restSigningRegion();
this.signingName = awsProperties.restSigningName();
this.credentialsProvider = awsProperties.restCredentialsProvider();
+ if (credentialsProvider instanceof AutoCloseable closeableCredentialsProvider) {
+ this.closeableGroup.addCloseable(closeableCredentialsProvider);
+ }
}
public AuthSession delegate() {
@@ -84,9 +97,14 @@ public HTTPRequest authenticate(HTTPRequest request) {
@Override
public void close() {
- delegate.close();
+ try {
+ closeableGroup.close();
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
}
+ @SuppressWarnings("deprecation")
private HTTPRequest sign(HTTPRequest request) {
Aws4SignerParams params =
Aws4SignerParams.builder()
diff --git a/aws/src/main/java/org/apache/iceberg/aws/S3FileIOAwsClientFactories.java b/aws/src/main/java/org/apache/iceberg/aws/S3FileIOAwsClientFactories.java
index 4aec0bda2a13..3306163baffd 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/S3FileIOAwsClientFactories.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/S3FileIOAwsClientFactories.java
@@ -32,7 +32,7 @@ private S3FileIOAwsClientFactories() {}
/**
* Attempts to load an AWS client factory class for S3 file IO defined in the catalog property
* {@link S3FileIOProperties#CLIENT_FACTORY}. If the property wasn't set, fallback to {@link
- * AwsClientFactories#from(Map) to intialize an AWS client factory class}
+ * AwsClientFactories#from(Map) to initialize an AWS client factory class}
*
* @param properties catalog properties
* @return an instance of a factory class
diff --git a/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java b/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java
index 273baa674804..fbd845852ca9 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/UrlConnectionHttpClientConfigurations.java
@@ -35,6 +35,8 @@ class UrlConnectionHttpClientConfigurations extends BaseHttpClientConfigurations
private Long httpClientUrlConnectionConnectionTimeoutMs;
private Long httpClientUrlConnectionSocketTimeoutMs;
private String proxyEndpoint;
+ private Boolean proxyUseSystemPropertyValues;
+ private Boolean proxyUseEnvironmentVariableValues;
private UrlConnectionHttpClientConfigurations() {}
@@ -56,6 +58,12 @@ private void initialize(Map httpClientProperties) {
this.proxyEndpoint =
PropertyUtil.propertyAsString(
httpClientProperties, HttpClientProperties.PROXY_ENDPOINT, null);
+ this.proxyUseSystemPropertyValues =
+ PropertyUtil.propertyAsNullableBoolean(
+ httpClientProperties, HttpClientProperties.PROXY_USE_SYSTEM_PROPERTY_VALUES);
+ this.proxyUseEnvironmentVariableValues =
+ PropertyUtil.propertyAsNullableBoolean(
+ httpClientProperties, HttpClientProperties.PROXY_USE_ENVIRONMENT_VARIABLE_VALUES);
}
@VisibleForTesting
@@ -69,9 +77,26 @@ void configureUrlConnectionHttpClientBuilder(
urlConnectionHttpClientBuilder.socketTimeout(
Duration.ofMillis(httpClientUrlConnectionSocketTimeoutMs));
}
- if (proxyEndpoint != null) {
- urlConnectionHttpClientBuilder.proxyConfiguration(
- ProxyConfiguration.builder().endpoint(URI.create(proxyEndpoint)).build());
+ configureProxy(urlConnectionHttpClientBuilder);
+ }
+
+ private void configureProxy(UrlConnectionHttpClient.Builder urlConnectionHttpClientBuilder) {
+ if (proxyEndpoint != null
+ || proxyUseSystemPropertyValues != null
+ || proxyUseEnvironmentVariableValues != null) {
+ ProxyConfiguration.Builder proxyBuilder = ProxyConfiguration.builder();
+
+ if (proxyEndpoint != null) {
+ proxyBuilder.endpoint(URI.create(proxyEndpoint));
+ }
+ if (proxyUseSystemPropertyValues != null) {
+ proxyBuilder.useSystemPropertyValues(proxyUseSystemPropertyValues);
+ }
+ if (proxyUseEnvironmentVariableValues != null) {
+ proxyBuilder.useEnvironmentVariablesValues(proxyUseEnvironmentVariableValues);
+ }
+
+ urlConnectionHttpClientBuilder.proxyConfiguration(proxyBuilder.build());
}
}
@@ -87,6 +112,8 @@ protected String generateHttpClientCacheKey() {
keyComponents.put("connectionTimeoutMs", httpClientUrlConnectionConnectionTimeoutMs);
keyComponents.put("socketTimeoutMs", httpClientUrlConnectionSocketTimeoutMs);
keyComponents.put("proxyEndpoint", proxyEndpoint);
+ keyComponents.put("proxyUseSystemPropertyValues", proxyUseSystemPropertyValues);
+ keyComponents.put("proxyUseEnvironmentVariableValues", proxyUseEnvironmentVariableValues);
return keyComponents.entrySet().stream()
.map(entry -> entry.getKey() + "=" + Objects.toString(entry.getValue(), "null"))
diff --git a/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java b/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java
index 0c991af75076..7c75f99d6d69 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/dynamodb/DynamoDbCatalog.java
@@ -53,6 +53,7 @@
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.util.LocationUtil;
+import org.apache.iceberg.util.PropertyUtil;
import org.apache.iceberg.util.Tasks;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,6 +113,7 @@ public class DynamoDbCatalog extends BaseMetastoreCatalog
private FileIO fileIO;
private CloseableGroup closeableGroup;
private Map catalogProperties;
+ private boolean uniqueTableLocation;
public DynamoDbCatalog() {}
@@ -123,12 +125,21 @@ public void initialize(String name, Map properties) {
properties.get(CatalogProperties.WAREHOUSE_LOCATION),
new AwsProperties(properties),
AwsClientFactories.from(properties).dynamo(),
- initializeFileIO(properties));
+ initializeFileIO(properties),
+ PropertyUtil.propertyAsBoolean(
+ properties,
+ CatalogProperties.UNIQUE_TABLE_LOCATION,
+ CatalogProperties.UNIQUE_TABLE_LOCATION_DEFAULT));
}
@VisibleForTesting
void initialize(
- String name, String path, AwsProperties properties, DynamoDbClient client, FileIO io) {
+ String name,
+ String path,
+ AwsProperties properties,
+ DynamoDbClient client,
+ FileIO io,
+ boolean uniqTableLocation) {
Preconditions.checkArgument(
!Strings.isNullOrEmpty(path),
"Cannot initialize DynamoDbCatalog because warehousePath must not be null or empty");
@@ -138,6 +149,7 @@ void initialize(
this.warehousePath = LocationUtil.stripTrailingSlash(path);
this.dynamo = client;
this.fileIO = io;
+ this.uniqueTableLocation = uniqTableLocation;
this.closeableGroup = new CloseableGroup();
closeableGroup.addCloseable(dynamo);
@@ -177,12 +189,12 @@ protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) {
}
String defaultLocationCol = toPropertyCol(PROPERTY_DEFAULT_LOCATION);
+ String tableLocation = LocationUtil.tableLocation(tableIdentifier, uniqueTableLocation);
if (response.item().containsKey(defaultLocationCol)) {
- return String.format(
- "%s/%s", response.item().get(defaultLocationCol).s(), tableIdentifier.name());
+ return String.format("%s/%s", response.item().get(defaultLocationCol).s(), tableLocation);
} else {
return String.format(
- "%s/%s.db/%s", warehousePath, tableIdentifier.namespace(), tableIdentifier.name());
+ "%s/%s.db/%s", warehousePath, tableIdentifier.namespace(), tableLocation);
}
}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java b/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java
index 47807a2b9f37..94e53cc1ab69 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/glue/GlueCatalog.java
@@ -89,6 +89,7 @@ public class GlueCatalog extends BaseMetastoreCatalog
private Object hadoopConf;
private String catalogName;
private String warehousePath;
+ private boolean uniqueTableLocation;
private AwsProperties awsProperties;
private S3FileIOProperties s3FileIOProperties;
private LockManager lockManager;
@@ -144,7 +145,11 @@ public void initialize(String name, Map properties) {
new AwsProperties(properties),
new S3FileIOProperties(properties),
awsClientFactory.glue(),
- initializeLockManager(properties));
+ initializeLockManager(properties),
+ PropertyUtil.propertyAsBoolean(
+ properties,
+ CatalogProperties.UNIQUE_TABLE_LOCATION,
+ CatalogProperties.UNIQUE_TABLE_LOCATION_DEFAULT));
}
private LockManager initializeLockManager(Map properties) {
@@ -172,7 +177,17 @@ void initialize(
LockManager lock,
Map catalogProps) {
this.catalogProperties = catalogProps;
- initialize(name, path, properties, s3Properties, client, lock);
+ initialize(
+ name,
+ path,
+ properties,
+ s3Properties,
+ client,
+ lock,
+ PropertyUtil.propertyAsBoolean(
+ catalogProps,
+ CatalogProperties.UNIQUE_TABLE_LOCATION,
+ CatalogProperties.UNIQUE_TABLE_LOCATION_DEFAULT));
}
@VisibleForTesting
@@ -182,13 +197,15 @@ void initialize(
AwsProperties properties,
S3FileIOProperties s3Properties,
GlueClient client,
- LockManager lock) {
+ LockManager lock,
+ boolean uniqTableLocation) {
this.catalogName = name;
this.awsProperties = properties;
this.s3FileIOProperties = s3Properties;
this.warehousePath = Strings.isNullOrEmpty(path) ? null : LocationUtil.stripTrailingSlash(path);
this.glue = client;
this.lockManager = lock;
+ this.uniqueTableLocation = uniqTableLocation;
this.closeableGroup = new CloseableGroup();
this.fileIOTracker = new FileIOTracker();
@@ -278,9 +295,10 @@ protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) {
tableIdentifier, awsProperties.glueCatalogSkipNameValidation()))
.build());
String dbLocationUri = response.database().locationUri();
+ String tableLocation = LocationUtil.tableLocation(tableIdentifier, uniqueTableLocation);
if (dbLocationUri != null) {
dbLocationUri = LocationUtil.stripTrailingSlash(dbLocationUri);
- return String.format("%s/%s", dbLocationUri, tableIdentifier.name());
+ return String.format("%s/%s", dbLocationUri, tableLocation);
}
ValidationException.check(
@@ -292,7 +310,7 @@ protected String defaultWarehouseLocation(TableIdentifier tableIdentifier) {
warehousePath,
IcebergToGlueConverter.getDatabaseName(
tableIdentifier, awsProperties.glueCatalogSkipNameValidation()),
- tableIdentifier.name());
+ tableLocation);
}
@Override
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorUtil.java b/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorUtil.java
index ed1c3a4879e3..36a0a235da5b 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorUtil.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/AnalyticsAcceleratorUtil.java
@@ -28,7 +28,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.services.s3.S3AsyncClient;
-import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import software.amazon.s3.analyticsaccelerator.ObjectClientConfiguration;
import software.amazon.s3.analyticsaccelerator.S3SdkObjectClient;
import software.amazon.s3.analyticsaccelerator.S3SeekableInputStream;
@@ -36,8 +35,6 @@
import software.amazon.s3.analyticsaccelerator.S3SeekableInputStreamFactory;
import software.amazon.s3.analyticsaccelerator.common.ConnectorConfiguration;
import software.amazon.s3.analyticsaccelerator.request.ObjectClient;
-import software.amazon.s3.analyticsaccelerator.request.ObjectMetadata;
-import software.amazon.s3.analyticsaccelerator.util.OpenStreamInformation;
import software.amazon.s3.analyticsaccelerator.util.S3URI;
class AnalyticsAcceleratorUtil {
@@ -58,15 +55,6 @@ private AnalyticsAcceleratorUtil() {}
public static SeekableInputStream newStream(S3InputFile inputFile) {
S3URI uri = S3URI.of(inputFile.uri().bucket(), inputFile.uri().key());
- HeadObjectResponse metadata = inputFile.getObjectMetadata();
- OpenStreamInformation openStreamInfo =
- OpenStreamInformation.builder()
- .objectMetadata(
- ObjectMetadata.builder()
- .contentLength(metadata.contentLength())
- .etag(metadata.eTag())
- .build())
- .build();
S3SeekableInputStreamFactory factory =
STREAM_FACTORY_CACHE.get(
@@ -74,7 +62,7 @@ public static SeekableInputStream newStream(S3InputFile inputFile) {
AnalyticsAcceleratorUtil::createNewFactory);
try {
- S3SeekableInputStream seekableInputStream = factory.createStream(uri, openStreamInfo);
+ S3SeekableInputStream seekableInputStream = factory.createStream(uri);
return new AnalyticsAcceleratorInputStreamWrapper(seekableInputStream);
} catch (IOException e) {
throw new RuntimeIOException(
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIO.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIO.java
index d5e51ed74ad6..50e507b0cd8b 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIO.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIO.java
@@ -18,16 +18,22 @@
*/
package org.apache.iceberg.aws.s3;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
import org.apache.iceberg.aws.S3FileIOAwsClientFactories;
@@ -94,7 +100,7 @@ public class S3FileIO
private static final String DEFAULT_METRICS_IMPL =
"org.apache.iceberg.hadoop.HadoopMetricsContext";
private static final String ROOT_PREFIX = "s3";
- private static volatile ExecutorService executorService;
+ private static volatile ScheduledExecutorService executorService;
private String credential = null;
private SerializableSupplier s3;
@@ -104,8 +110,9 @@ public class S3FileIO
private final AtomicBoolean isResourceClosed = new AtomicBoolean(false);
private transient StackTraceElement[] createStack;
// use modifiable collection for Kryo serde
- private List storageCredentials = Lists.newArrayList();
+ private volatile List storageCredentials = Lists.newArrayList();
private transient volatile Map clientByPrefix;
+ private transient volatile ScheduledFuture refreshFuture;
/**
* No-arg constructor to load the FileIO dynamically.
@@ -419,7 +426,11 @@ private Map clientByPrefix() {
new PrefixedS3Client(
storageCredential.prefix(), propertiesWithCredentials, s3, s3Async));
});
+
this.clientByPrefix = localClientByPrefix;
+ // Note: the s3 clients separately refresh via the VendedCredentialsProvider but are
+ // not directly referencable from the FileIO
+ scheduleCredentialRefresh();
}
}
}
@@ -427,14 +438,54 @@ private Map clientByPrefix() {
return clientByPrefix;
}
- private ExecutorService executorService() {
+ private void scheduleCredentialRefresh() {
+ storageCredentials.stream()
+ .map(
+ storageCredential ->
+ storageCredential.config().get(S3FileIOProperties.SESSION_TOKEN_EXPIRES_AT_MS))
+ .filter(Objects::nonNull)
+ .map(expiresAtString -> Instant.ofEpochMilli(Long.parseLong(expiresAtString)))
+ .min(Comparator.naturalOrder())
+ .ifPresent(
+ expiresAt -> {
+ Instant prefetchAt = expiresAt.minus(5, ChronoUnit.MINUTES);
+ long delay = Duration.between(Instant.now(), prefetchAt).toMillis();
+ this.refreshFuture =
+ executorService()
+ .schedule(this::refreshStorageCredentials, delay, TimeUnit.MILLISECONDS);
+ });
+ }
+
+ private void refreshStorageCredentials() {
+ if (isResourceClosed.get()) {
+ return;
+ }
+
+ try (VendedCredentialsProvider provider = VendedCredentialsProvider.create(properties)) {
+ List refreshed =
+ provider.fetchCredentials().credentials().stream()
+ .filter(c -> c.prefix().startsWith(ROOT_PREFIX))
+ .map(c -> StorageCredential.create(c.prefix(), c.config()))
+ .collect(Collectors.toList());
+
+ if (!refreshed.isEmpty() && !isResourceClosed.get()) {
+ this.storageCredentials = Lists.newArrayList(refreshed);
+ scheduleCredentialRefresh();
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed to refresh storage credentials", e);
+ }
+ }
+
+ private ScheduledExecutorService executorService() {
if (executorService == null) {
synchronized (S3FileIO.class) {
if (executorService == null) {
executorService =
- ThreadPools.newExitingWorkerPool(
- "iceberg-s3fileio-delete",
- clientForStoragePath(ROOT_PREFIX).s3FileIOProperties().deleteThreads());
+ ThreadPools.newExitingScheduledPool(
+ "iceberg-s3fileio-tasks",
+ clientForStoragePath(ROOT_PREFIX).s3FileIOProperties().deleteThreads(),
+ Duration.ofSeconds(10));
}
}
}
@@ -491,10 +542,14 @@ public void close() {
clientByPrefix.values().forEach(PrefixedS3Client::close);
this.clientByPrefix = null;
}
+ if (refreshFuture != null) {
+ refreshFuture.cancel(true);
+ refreshFuture = null;
+ }
}
}
- @SuppressWarnings({"checkstyle:NoFinalizer", "Finalize"})
+ @SuppressWarnings({"checkstyle:NoFinalizer", "Finalize", "deprecation"})
@Override
protected void finalize() throws Throwable {
super.finalize();
@@ -559,8 +614,21 @@ private boolean recoverObject(PrefixedS3Client client, ObjectVersion version, St
@Override
public void setCredentials(List credentials) {
Preconditions.checkArgument(credentials != null, "Invalid storage credentials: null");
+ // stop any refresh that might be scheduled
+ if (refreshFuture != null) {
+ refreshFuture.cancel(true);
+ }
+
// copy credentials into a modifiable collection for Kryo serde
this.storageCredentials = Lists.newArrayList(credentials);
+
+ // if the clients are already initialized, we need to close and allow them to be recreated
+ synchronized (this) {
+ if (clientByPrefix != null) {
+ clientByPrefix.values().forEach(PrefixedS3Client::close);
+ this.clientByPrefix = null;
+ }
+ }
}
@Override
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java
index 6bf582f00bbb..922010d61d27 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java
@@ -270,7 +270,7 @@ public class S3FileIOProperties implements Serializable {
* This expiration time is currently only used in {@link VendedCredentialsProvider} for refreshing
* vended credentials.
*/
- static final String SESSION_TOKEN_EXPIRES_AT_MS = "s3.session-token-expires-at-ms";
+ public static final String SESSION_TOKEN_EXPIRES_AT_MS = "s3.session-token-expires-at-ms";
/**
* Enable to make S3FileIO, to make cross-region call to the region specified in the ARN of an
@@ -295,6 +295,18 @@ public class S3FileIOProperties implements Serializable {
public static final boolean REMOTE_SIGNING_ENABLED_DEFAULT = false;
+ /**
+ * Enables or disables chunked encoding for S3 requests.
+ *
+ * This feature is enabled by default to match the AWS SDK default behavior.
+ *
+ *
For more details see:
+ * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Configuration.html#chunkedEncodingEnabled()
+ */
+ public static final String CHUNKED_ENCODING_ENABLED = "s3.chunked-encoding-enabled";
+
+ public static final boolean CHUNKED_ENCODING_ENABLED_DEFAULT = true;
+
/** Configure the batch size used when deleting multiple files from a given S3 bucket */
public static final String DELETE_BATCH_SIZE = "s3.delete.batch-size";
@@ -509,6 +521,7 @@ public class S3FileIOProperties implements Serializable {
private String stagingDirectory;
private ObjectCannedACL acl;
private boolean isChecksumEnabled;
+ private boolean isChunkedEncodingEnabled;
private final Set writeTags;
private boolean isWriteTableTagEnabled;
private boolean isWriteNamespaceTagEnabled;
@@ -551,6 +564,7 @@ public S3FileIOProperties() {
this.deleteBatchSize = DELETE_BATCH_SIZE_DEFAULT;
this.stagingDirectory = System.getProperty("java.io.tmpdir");
this.isChecksumEnabled = CHECKSUM_ENABLED_DEFAULT;
+ this.isChunkedEncodingEnabled = CHUNKED_ENCODING_ENABLED_DEFAULT;
this.writeTags = Sets.newHashSet();
this.isWriteTableTagEnabled = WRITE_TABLE_TAG_ENABLED_DEFAULT;
this.isWriteNamespaceTagEnabled = WRITE_NAMESPACE_TAG_ENABLED_DEFAULT;
@@ -641,6 +655,9 @@ public S3FileIOProperties(Map properties) {
"Cannot support S3 CannedACL " + aclType);
this.isChecksumEnabled =
PropertyUtil.propertyAsBoolean(properties, CHECKSUM_ENABLED, CHECKSUM_ENABLED_DEFAULT);
+ this.isChunkedEncodingEnabled =
+ PropertyUtil.propertyAsBoolean(
+ properties, CHUNKED_ENCODING_ENABLED, CHUNKED_ENCODING_ENABLED_DEFAULT);
this.deleteBatchSize =
PropertyUtil.propertyAsInt(properties, DELETE_BATCH_SIZE, DELETE_BATCH_SIZE_DEFAULT);
Preconditions.checkArgument(
@@ -808,6 +825,10 @@ public boolean isChecksumEnabled() {
return this.isChecksumEnabled;
}
+ public boolean isChunkedEncodingEnabled() {
+ return this.isChunkedEncodingEnabled;
+ }
+
public boolean isRemoteSigningEnabled() {
return this.isRemoteSigningEnabled;
}
@@ -994,6 +1015,7 @@ public void applyServiceConfigurations(T builder) {
.pathStyleAccessEnabled(isPathStyleAccess)
.useArnRegionEnabled(isUseArnRegionEnabled)
.accelerateModeEnabled(isAccelerationEnabled)
+ .chunkedEncodingEnabled(isChunkedEncodingEnabled)
.build());
}
@@ -1006,6 +1028,7 @@ public void applyServiceConfigurations(T builder) {
* S3Client.builder().applyMutation(s3FileIOProperties::applyS3SignerConfiguration)
*
*/
+ @SuppressWarnings("deprecation")
public void applySignerConfiguration(T builder) {
if (isRemoteSigningEnabled) {
ClientOverrideConfiguration.Builder configBuilder =
@@ -1060,6 +1083,7 @@ public void applyEndpointConfigurations(T bu
* S3Client.builder().applyMutation(s3FileIOProperties::applyRetryConfigurations)
*
*/
+ @SuppressWarnings("deprecation")
public void applyRetryConfigurations(T builder) {
ClientOverrideConfiguration.Builder configBuilder =
null != builder.overrideConfiguration()
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3InputStream.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3InputStream.java
index 4d37ac333030..9c91cc58f8d5 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3InputStream.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3InputStream.java
@@ -290,7 +290,7 @@ public void setSkipSize(int skipSize) {
this.skipSize = skipSize;
}
- @SuppressWarnings({"checkstyle:NoFinalizer", "Finalize"})
+ @SuppressWarnings({"checkstyle:NoFinalizer", "Finalize", "deprecation"})
@Override
protected void finalize() throws Throwable {
super.finalize();
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3OutputStream.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3OutputStream.java
index 684e6549a86b..50963127f52a 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3OutputStream.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3OutputStream.java
@@ -480,7 +480,7 @@ private void createStagingDirectoryIfNotExists() throws IOException, SecurityExc
}
}
- @SuppressWarnings({"checkstyle:NoFinalizer", "Finalize"})
+ @SuppressWarnings({"checkstyle:NoFinalizer", "Finalize", "deprecation"})
@Override
protected void finalize() throws Throwable {
super.finalize();
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/VendedCredentialsProvider.java b/aws/src/main/java/org/apache/iceberg/aws/s3/VendedCredentialsProvider.java
index fc42bd789859..75d114d4efbe 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/VendedCredentialsProvider.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/VendedCredentialsProvider.java
@@ -29,7 +29,9 @@
import org.apache.iceberg.relocated.com.google.common.base.Strings;
import org.apache.iceberg.rest.ErrorHandlers;
import org.apache.iceberg.rest.HTTPClient;
+import org.apache.iceberg.rest.RESTCatalogProperties;
import org.apache.iceberg.rest.RESTClient;
+import org.apache.iceberg.rest.RESTUtil;
import org.apache.iceberg.rest.auth.AuthManager;
import org.apache.iceberg.rest.auth.AuthManagers;
import org.apache.iceberg.rest.auth.AuthSession;
@@ -50,6 +52,7 @@ public class VendedCredentialsProvider implements AwsCredentialsProvider, SdkAut
private final CachedSupplier credentialCache;
private final String catalogEndpoint;
private final String credentialsEndpoint;
+ private final String planId;
private AuthManager authManager;
private AuthSession authSession;
@@ -65,6 +68,7 @@ private VendedCredentialsProvider(Map properties) {
.build();
this.catalogEndpoint = properties.get(CatalogProperties.URI);
this.credentialsEndpoint = properties.get(URI);
+ this.planId = properties.getOrDefault(RESTCatalogProperties.REST_SCAN_PLAN_ID, null);
}
@Override
@@ -89,7 +93,11 @@ private RESTClient httpClient() {
synchronized (this) {
if (null == client) {
authManager = AuthManagers.loadAuthManager("s3-credentials-refresh", properties);
- HTTPClient httpClient = HTTPClient.builder(properties).uri(catalogEndpoint).build();
+ HTTPClient httpClient =
+ HTTPClient.builder(properties)
+ .uri(catalogEndpoint)
+ .withHeaders(RESTUtil.configHeaders(properties))
+ .build();
authSession = authManager.catalogSession(httpClient, properties);
client = httpClient.withAuthSession(authSession);
}
@@ -99,11 +107,11 @@ private RESTClient httpClient() {
return client;
}
- private LoadCredentialsResponse fetchCredentials() {
+ LoadCredentialsResponse fetchCredentials() {
return httpClient()
.get(
credentialsEndpoint,
- null,
+ null != planId ? Map.of("planId", planId) : null,
LoadCredentialsResponse.class,
Map.of(),
ErrorHandlers.defaultErrorHandler());
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java
index 89145b2465e5..7f1d6c3cc848 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java
@@ -40,6 +40,10 @@
import org.apache.iceberg.rest.responses.ErrorResponse;
import org.apache.iceberg.rest.responses.OAuthTokenResponse;
+/**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@code RESTObjectMapper} instead.
+ */
+@Deprecated
public class S3ObjectMapper {
private static final JsonFactory FACTORY = new JsonFactory();
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequest.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequest.java
index 879ce8599352..995f6e7e4860 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequest.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequest.java
@@ -18,31 +18,13 @@
*/
package org.apache.iceberg.aws.s3.signer;
-import java.net.URI;
-import java.util.List;
-import java.util.Map;
-import javax.annotation.Nullable;
-import org.apache.iceberg.rest.RESTRequest;
+import org.apache.iceberg.rest.requests.RemoteSignRequest;
import org.immutables.value.Value;
+/**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link RemoteSignRequest} instead.
+ */
+@Deprecated
@Value.Immutable
-public interface S3SignRequest extends RESTRequest {
- String region();
-
- String method();
-
- URI uri();
-
- Map> headers();
-
- Map properties();
-
- @Value.Default
- @Nullable
- default String body() {
- return null;
- }
-
- @Override
- default void validate() {}
-}
+@SuppressWarnings("immutables:subtype")
+public interface S3SignRequest extends RemoteSignRequest {}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequestParser.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequestParser.java
index efb11b3cdf55..5d2a7d684460 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequestParser.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequestParser.java
@@ -21,108 +21,47 @@
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.IOException;
-import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.apache.iceberg.util.JsonUtil;
+import org.apache.iceberg.rest.requests.RemoteSignRequest;
+import org.apache.iceberg.rest.requests.RemoteSignRequestParser;
+/**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link RemoteSignRequestParser} instead.
+ */
+@Deprecated
public class S3SignRequestParser {
- private static final String REGION = "region";
- private static final String METHOD = "method";
- private static final String URI = "uri";
- private static final String HEADERS = "headers";
- private static final String PROPERTIES = "properties";
- private static final String BODY = "body";
-
private S3SignRequestParser() {}
public static String toJson(S3SignRequest request) {
- return toJson(request, false);
+ return RemoteSignRequestParser.toJson(request, false);
}
public static String toJson(S3SignRequest request, boolean pretty) {
- return JsonUtil.generate(gen -> toJson(request, gen), pretty);
+ return RemoteSignRequestParser.toJson(request, pretty);
}
public static void toJson(S3SignRequest request, JsonGenerator gen) throws IOException {
- Preconditions.checkArgument(null != request, "Invalid s3 sign request: null");
-
- gen.writeStartObject();
-
- gen.writeStringField(REGION, request.region());
- gen.writeStringField(METHOD, request.method());
- gen.writeStringField(URI, request.uri().toString());
- headersToJson(HEADERS, request.headers(), gen);
-
- if (!request.properties().isEmpty()) {
- JsonUtil.writeStringMap(PROPERTIES, request.properties(), gen);
- }
-
- if (request.body() != null && !request.body().isEmpty()) {
- gen.writeStringField(BODY, request.body());
- }
-
- gen.writeEndObject();
+ RemoteSignRequestParser.toJson(request, gen);
}
public static S3SignRequest fromJson(String json) {
- return JsonUtil.parse(json, S3SignRequestParser::fromJson);
+ RemoteSignRequest request = RemoteSignRequestParser.fromJson(json);
+ return ImmutableS3SignRequest.builder().from(request).build();
}
public static S3SignRequest fromJson(JsonNode json) {
- Preconditions.checkArgument(null != json, "Cannot parse s3 sign request from null object");
- Preconditions.checkArgument(
- json.isObject(), "Cannot parse s3 sign request from non-object: %s", json);
-
- String region = JsonUtil.getString(REGION, json);
- String method = JsonUtil.getString(METHOD, json);
- java.net.URI uri = java.net.URI.create(JsonUtil.getString(URI, json));
- Map> headers = headersFromJson(HEADERS, json);
-
- ImmutableS3SignRequest.Builder builder =
- ImmutableS3SignRequest.builder().region(region).method(method).uri(uri).headers(headers);
-
- if (json.has(PROPERTIES)) {
- builder.properties(JsonUtil.getStringMap(PROPERTIES, json));
- }
-
- if (json.has(BODY)) {
- builder.body(JsonUtil.getString(BODY, json));
- }
-
- return builder.build();
+ RemoteSignRequest request = RemoteSignRequestParser.fromJson(json);
+ return ImmutableS3SignRequest.builder().from(request).build();
}
static void headersToJson(String property, Map> headers, JsonGenerator gen)
throws IOException {
- gen.writeObjectFieldStart(property);
- for (Entry> entry : headers.entrySet()) {
- gen.writeFieldName(entry.getKey());
-
- gen.writeStartArray();
- for (String val : entry.getValue()) {
- gen.writeString(val);
- }
- gen.writeEndArray();
- }
- gen.writeEndObject();
+ RemoteSignRequestParser.headersToJson(property, headers, gen);
}
static Map> headersFromJson(String property, JsonNode json) {
- Map> headers = Maps.newHashMap();
- JsonNode headersNode = JsonUtil.get(property, json);
- headersNode
- .fields()
- .forEachRemaining(
- entry -> {
- String key = entry.getKey();
- List values = Arrays.asList(JsonUtil.getStringArray(entry.getValue()));
- headers.put(key, values);
- });
- return headers;
+ return RemoteSignRequestParser.headersFromJson(property, json);
}
}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponse.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponse.java
index 40c2059488f8..6fbaa90fe7af 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponse.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponse.java
@@ -18,18 +18,13 @@
*/
package org.apache.iceberg.aws.s3.signer;
-import java.net.URI;
-import java.util.List;
-import java.util.Map;
-import org.apache.iceberg.rest.RESTResponse;
+import org.apache.iceberg.rest.responses.RemoteSignResponse;
import org.immutables.value.Value;
+/**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link RemoteSignResponse} instead.
+ */
+@Deprecated
@Value.Immutable
-public interface S3SignResponse extends RESTResponse {
- URI uri();
-
- Map> headers();
-
- @Override
- default void validate() {}
-}
+@SuppressWarnings("immutables:subtype")
+public interface S3SignResponse extends RemoteSignResponse {}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponseParser.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponseParser.java
index 69d6de8f04ac..be63a51b38fb 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponseParser.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponseParser.java
@@ -21,49 +21,37 @@
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.util.JsonUtil;
+import org.apache.iceberg.rest.responses.RemoteSignResponse;
+import org.apache.iceberg.rest.responses.RemoteSignResponseParser;
+/**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link RemoteSignResponseParser}
+ * instead.
+ */
+@Deprecated
public class S3SignResponseParser {
- private static final String URI = "uri";
- private static final String HEADERS = "headers";
-
private S3SignResponseParser() {}
- public static String toJson(S3SignResponse request) {
- return toJson(request, false);
+ public static String toJson(S3SignResponse response) {
+ return RemoteSignResponseParser.toJson(response, false);
}
- public static String toJson(S3SignResponse request, boolean pretty) {
- return JsonUtil.generate(gen -> toJson(request, gen), pretty);
+ public static String toJson(S3SignResponse response, boolean pretty) {
+ return RemoteSignResponseParser.toJson(response, pretty);
}
public static void toJson(S3SignResponse response, JsonGenerator gen) throws IOException {
- Preconditions.checkArgument(null != response, "Invalid s3 sign response: null");
-
- gen.writeStartObject();
-
- gen.writeStringField(URI, response.uri().toString());
- S3SignRequestParser.headersToJson(HEADERS, response.headers(), gen);
-
- gen.writeEndObject();
+ RemoteSignResponseParser.toJson(response, gen);
}
public static S3SignResponse fromJson(String json) {
- return JsonUtil.parse(json, S3SignResponseParser::fromJson);
+ RemoteSignResponse result = RemoteSignResponseParser.fromJson(json);
+ return ImmutableS3SignResponse.builder().from(result).build();
}
public static S3SignResponse fromJson(JsonNode json) {
- Preconditions.checkArgument(null != json, "Cannot parse s3 sign response from null object");
- Preconditions.checkArgument(
- json.isObject(), "Cannot parse s3 sign response from non-object: %s", json);
-
- java.net.URI uri = java.net.URI.create(JsonUtil.getString(URI, json));
- Map> headers = S3SignRequestParser.headersFromJson(HEADERS, json);
-
- return ImmutableS3SignResponse.builder().uri(uri).headers(headers).build();
+ RemoteSignResponse result = RemoteSignResponseParser.fromJson(json);
+ return ImmutableS3SignResponse.builder().from(result).build();
}
}
diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java
index 6385d8875d22..7a463abd3d2d 100644
--- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java
+++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java
@@ -37,6 +37,7 @@
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.rest.ErrorHandlers;
import org.apache.iceberg.rest.HTTPClient;
+import org.apache.iceberg.rest.RESTCatalogProperties;
import org.apache.iceberg.rest.RESTClient;
import org.apache.iceberg.rest.RESTUtil;
import org.apache.iceberg.rest.ResourcePaths;
@@ -45,6 +46,9 @@
import org.apache.iceberg.rest.auth.AuthSession;
import org.apache.iceberg.rest.auth.OAuth2Properties;
import org.apache.iceberg.rest.auth.OAuth2Util;
+import org.apache.iceberg.rest.requests.ImmutableRemoteSignRequest;
+import org.apache.iceberg.rest.requests.RemoteSignRequest;
+import org.apache.iceberg.rest.responses.RemoteSignResponse;
import org.apache.iceberg.util.PropertyUtil;
import org.immutables.value.Value;
import org.slf4j.Logger;
@@ -64,13 +68,30 @@ public abstract class S3V4RestSignerClient
extends AbstractAws4Signer implements AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(S3V4RestSignerClient.class);
- public static final String S3_SIGNER_URI = "s3.signer.uri";
- public static final String S3_SIGNER_ENDPOINT = "s3.signer.endpoint";
- static final String S3_SIGNER_DEFAULT_ENDPOINT = "v1/aws/s3/sign";
- static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD";
- static final String CACHE_CONTROL = "Cache-Control";
- static final String CACHE_CONTROL_PRIVATE = "private";
- static final String CACHE_CONTROL_NO_CACHE = "no-cache";
+
+ public static final String S3_PROVIDER = "s3";
+
+ /**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link
+ * RESTCatalogProperties#SIGNER_URI} instead.
+ */
+ @Deprecated public static final String S3_SIGNER_URI = "s3.signer.uri";
+
+ /**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link
+ * RESTCatalogProperties#SIGNER_URI} instead.
+ */
+ @Deprecated public static final String S3_SIGNER_ENDPOINT = "s3.signer.endpoint";
+
+ /**
+ * @deprecated since 1.11.0, will be removed in 1.12.0; there is no replacement.
+ */
+ @Deprecated static final String S3_SIGNER_DEFAULT_ENDPOINT = "v1/aws/s3/sign";
+
+ @VisibleForTesting static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD";
+
+ private static final String CACHE_CONTROL = "Cache-Control";
+ private static final String CACHE_CONTROL_PRIVATE = "private";
private static final Cache SIGNED_COMPONENT_CACHE =
Caffeine.newBuilder().expireAfterWrite(30, TimeUnit.SECONDS).maximumSize(100).build();
@@ -94,13 +115,28 @@ public Supplier> requestPropertiesSupplier() {
@Value.Lazy
public String baseSignerUri() {
- return properties().getOrDefault(S3_SIGNER_URI, properties().get(CatalogProperties.URI));
+ // TODO remove in 1.12.0
+ if (properties().containsKey(S3_SIGNER_URI)) {
+ return properties().get(S3_SIGNER_URI);
+ }
+
+ return properties()
+ .getOrDefault(RESTCatalogProperties.SIGNER_URI, properties().get(CatalogProperties.URI));
}
@Value.Lazy
public String endpoint() {
- return RESTUtil.resolveEndpoint(
- baseSignerUri(), properties().getOrDefault(S3_SIGNER_ENDPOINT, S3_SIGNER_DEFAULT_ENDPOINT));
+ // TODO remove in 1.12.0
+ String endpointPath;
+ if (properties().containsKey(S3_SIGNER_ENDPOINT)) {
+ endpointPath = properties().get(S3_SIGNER_ENDPOINT);
+ } else {
+ endpointPath =
+ properties()
+ .getOrDefault(RESTCatalogProperties.SIGNER_ENDPOINT, S3_SIGNER_DEFAULT_ENDPOINT);
+ }
+
+ return RESTUtil.resolveEndpoint(baseSignerUri(), endpointPath);
}
/** A credential to exchange for a token in the OAuth2 client credentials flow. */
@@ -158,7 +194,9 @@ private RESTClient httpClient() {
// Don't include a base URI because this client may be used for contacting different
// catalogs.
httpClient =
- HTTPClient.builder(properties()).withObjectMapper(S3ObjectMapper.mapper()).build();
+ HTTPClient.builder(properties())
+ .withHeaders(RESTUtil.configHeaders(properties()))
+ .build();
}
}
}
@@ -194,8 +232,36 @@ private boolean credentialProvided() {
@Value.Check
protected void check() {
Preconditions.checkArgument(
- properties().containsKey(S3_SIGNER_URI) || properties().containsKey(CatalogProperties.URI),
+ properties().containsKey(S3_SIGNER_URI)
+ || properties().containsKey(RESTCatalogProperties.SIGNER_URI)
+ || properties().containsKey(CatalogProperties.URI),
"S3 signer service URI is required");
+
+ if (properties().containsKey(S3_SIGNER_URI)
+ && !properties().containsKey(RESTCatalogProperties.SIGNER_URI)) {
+ LOG.warn(
+ "S3 signer URI is configured via deprecated property {}, this won't be supported in future releases. "
+ + "Please use {} instead.",
+ S3_SIGNER_URI,
+ RESTCatalogProperties.SIGNER_URI);
+ }
+
+ if (properties().containsKey(S3_SIGNER_ENDPOINT)
+ && !properties().containsKey(RESTCatalogProperties.SIGNER_ENDPOINT)) {
+ LOG.warn(
+ "Signer endpoint is configured via deprecated property {}, this won't be supported in future releases. "
+ + "Please use {} instead.",
+ S3_SIGNER_ENDPOINT,
+ RESTCatalogProperties.SIGNER_ENDPOINT);
+ }
+
+ // TODO change to required in 1.12.0
+ if (!properties().containsKey(S3_SIGNER_ENDPOINT)
+ && !properties().containsKey(RESTCatalogProperties.SIGNER_ENDPOINT)) {
+ LOG.warn(
+ "Signer endpoint is not set, this won't be supported in future releases. Using deprecated default: {}",
+ S3_SIGNER_DEFAULT_ENDPOINT);
+ }
}
@Override
@@ -208,6 +274,7 @@ protected void processRequestPayload(
checkSignerParams(signerParams);
}
+ @SuppressWarnings("deprecation")
@Override
protected void processRequestPayload(
SdkHttpFullRequest.Builder mutableRequest,
@@ -237,14 +304,15 @@ public SdkHttpFullRequest sign(
AwsS3V4SignerParams signerParams =
extractSignerParams(AwsS3V4SignerParams.builder(), executionAttributes).build();
- S3SignRequest remoteSigningRequest =
- ImmutableS3SignRequest.builder()
+ RemoteSignRequest remoteSigningRequest =
+ ImmutableRemoteSignRequest.builder()
.method(request.method().name())
.region(signerParams.signingRegion().id())
.uri(request.getUri())
.headers(request.headers())
.properties(requestPropertiesSupplier().get())
.body(bodyAsString(request))
+ .provider(S3_PROVIDER)
.build();
Key cacheKey = Key.from(remoteSigningRequest);
@@ -256,21 +324,21 @@ public SdkHttpFullRequest sign(
} else {
Map responseHeaders = Maps.newHashMap();
Consumer> responseHeadersConsumer = responseHeaders::putAll;
- S3SignResponse s3SignResponse =
+ RemoteSignResponse remoteSignResponse =
httpClient()
.withAuthSession(authSession())
.post(
endpoint(),
remoteSigningRequest,
- S3SignResponse.class,
+ RemoteSignResponse.class,
Map.of(),
ErrorHandlers.defaultErrorHandler(),
responseHeadersConsumer);
signedComponent =
ImmutableSignedComponent.builder()
- .headers(s3SignResponse.headers())
- .signedURI(s3SignResponse.uri())
+ .headers(remoteSignResponse.headers())
+ .signedURI(remoteSignResponse.uri())
.build();
if (canBeCached(responseHeaders)) {
@@ -347,7 +415,7 @@ interface Key {
String uri();
- static Key from(S3SignRequest request) {
+ static Key from(RemoteSignRequest request) {
return ImmutableKey.builder()
.method(request.method())
.region(request.region())
diff --git a/aws/src/main/resources/s3-signer-open-api.yaml b/aws/src/main/resources/s3-signer-open-api.yaml
index 3d719c515b2a..0b98fcc59eff 100644
--- a/aws/src/main/resources/s3-signer-open-api.yaml
+++ b/aws/src/main/resources/s3-signer-open-api.yaml
@@ -17,19 +17,22 @@
# under the License.
#
+# ⚠️ WARNING: this API is deprecated. Use the new remote signing endpoint instead,
+# see open-api/rest-catalog-open-api.yaml.
+
---
openapi: 3.0.3
info:
- title: Apache Iceberg S3 Signer API
+ title: "[DEPRECATED] Apache Iceberg S3 Signer API"
license:
name: Apache 2.0
url: https://www.apache.org/licenses/LICENSE-2.0.html
version: 0.0.1
description:
- Defines the specification for the S3 Signer API.
+ "[DEPRECATED] Defines the specification for the S3 Signer API."
servers:
- url: "{scheme}://{host}/{basePath}"
- description: Server URL when the port can be inferred from the scheme
+ description: "[DEPRECATED] Server URL when the port can be inferred from the scheme"
variables:
scheme:
description: The scheme of the URI, either http or https.
@@ -41,7 +44,7 @@ servers:
description: Optional prefix to be prepended to all routes
default: ""
- url: "{scheme}://{host}:{port}/{basePath}"
- description: Generic base server URL, with all parts configurable
+ description: "[DEPRECATED] Generic base server URL, with all parts configurable"
variables:
scheme:
description: The scheme of the URI, either http or https.
@@ -61,9 +64,10 @@ paths:
/v1/aws/s3/sign:
post:
+ deprecated: true
tags:
- S3 Signer API
- summary: Remotely signs S3 requests
+ summary: "[DEPRECATED] Remotely signs S3 requests"
operationId: signS3Request
requestBody:
description: The request containing the headers to be signed
@@ -95,6 +99,7 @@ components:
schemas:
S3Headers:
+ deprecated: true
type: object
additionalProperties:
type: array
@@ -102,6 +107,7 @@ components:
type: string
S3SignRequest:
+ deprecated: true
required:
- region
- uri
@@ -133,7 +139,8 @@ components:
responses:
S3SignResponse:
- description: The response containing signed & unsigned headers. The server will also send
+ description: >
+ [DEPRECATED] The response containing signed & unsigned headers. The server will also send
a Cache-Control header, indicating whether the response can be cached (Cache-Control = ["private"])
or not (Cache-Control = ["no-cache"]).
content:
diff --git a/aws/src/test/java/org/apache/iceberg/aws/TestAwsClientFactories.java b/aws/src/test/java/org/apache/iceberg/aws/TestAwsClientFactories.java
index fe95f9364673..94ef60e05d68 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/TestAwsClientFactories.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/TestAwsClientFactories.java
@@ -23,6 +23,7 @@
import java.io.IOException;
import java.util.Map;
+import java.util.Optional;
import org.apache.iceberg.TestHelpers;
import org.apache.iceberg.aws.lakeformation.LakeFormationAwsClientFactory;
import org.apache.iceberg.aws.s3.S3FileIOProperties;
@@ -38,7 +39,10 @@
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.awscore.AwsClient;
import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.retries.api.RetryStrategy;
+import software.amazon.awssdk.retries.internal.DefaultAdaptiveRetryStrategy;
import software.amazon.awssdk.services.dynamodb.DynamoDbClient;
import software.amazon.awssdk.services.glue.GlueClient;
import software.amazon.awssdk.services.glue.model.GetTablesRequest;
@@ -245,6 +249,42 @@ public void testWithClassDoesNotImplementCredentialsProvider() {
testProviderAndAssertThrownBy(providerClassName, containsMessage);
}
+ @Test
+ public void testGlueClientSetsAdaptiveRetryPolicy() {
+ AwsClientFactory factory =
+ getAwsClientFactoryByCredentialsProvider(DummyValidProvider.class.getName());
+ GlueClient glueClient = factory.glue();
+ assertAwsClientSetsAdaptiveRetryPolicy(glueClient);
+ }
+
+ @Test
+ public void testKmsClientSetsAdaptiveRetryPolicy() {
+ AwsClientFactory factory =
+ getAwsClientFactoryByCredentialsProvider(DummyValidProvider.class.getName());
+ KmsClient kmsClient = factory.kms();
+ assertAwsClientSetsAdaptiveRetryPolicy(kmsClient);
+ }
+
+ @Test
+ public void testDynamoClientSetsAdaptiveRetryPolicy() {
+ AwsClientFactory factory =
+ getAwsClientFactoryByCredentialsProvider(DummyValidProvider.class.getName());
+ DynamoDbClient dynamoClient = factory.dynamo();
+ assertAwsClientSetsAdaptiveRetryPolicy(dynamoClient);
+ }
+
+ /**
+ * Extract the retry strategy from an AwsClient object, then assert that it's set to the correct
+ * strategy. This enforces that we correctly applied the retry configurations to the client
+ * object.
+ */
+ private void assertAwsClientSetsAdaptiveRetryPolicy(AwsClient client) {
+ Optional retryStrategy =
+ client.serviceClientConfiguration().overrideConfiguration().retryStrategy();
+ assertThat(retryStrategy).isPresent();
+ assertThat(retryStrategy.get()).isInstanceOf(DefaultAdaptiveRetryStrategy.class);
+ }
+
private void testProviderAndAssertThrownBy(String providerClassName, String containsMessage) {
AwsClientFactory defaultAwsClientFactory =
getAwsClientFactoryByCredentialsProvider(providerClassName);
diff --git a/aws/src/test/java/org/apache/iceberg/aws/TestAwsClientProperties.java b/aws/src/test/java/org/apache/iceberg/aws/TestAwsClientProperties.java
index 8c177ab5274e..47bb12c83b40 100644
--- a/aws/src/test/java/org/apache/iceberg/aws/TestAwsClientProperties.java
+++ b/aws/src/test/java/org/apache/iceberg/aws/TestAwsClientProperties.java
@@ -30,6 +30,7 @@
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
@@ -39,6 +40,17 @@
public class TestAwsClientProperties {
+ public static class CustomCredentialProvider implements AwsCredentialsProvider {
+ public static AwsCredentialsProvider create(Map properties) {
+ return new CustomCredentialProvider();
+ }
+
+ @Override
+ public AwsCredentials resolveCredentials() {
+ return AwsBasicCredentials.builder().build();
+ }
+ }
+
@Test
public void testApplyClientRegion() {
Map properties = Maps.newHashMap();
@@ -180,7 +192,7 @@ public void refreshCredentialsEndpointWithOverridingOAuthToken() {
Map properties =
ImmutableMap.of(
CatalogProperties.URI,
- "http://localhost:1234/v1",
+ "http://localhost:1234/v1/catalog",
AwsClientProperties.REFRESH_CREDENTIALS_ENDPOINT,
"http://localhost:1234/v1/credentials",
OAuth2Properties.TOKEN,
@@ -189,17 +201,22 @@ public void refreshCredentialsEndpointWithOverridingOAuthToken() {
"specific-token");
AwsClientProperties awsClientProperties = new AwsClientProperties(properties);
- Map