Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
- `TableCleanupTaskHandler` now clamps `TABLE_METADATA_CLEANUP_BATCH_SIZE` to at least 1. Previously a non-positive realm override caused an infinite loop (0) or `IllegalArgumentException` (<0) when splitting metadata files for cleanup.
- Azure SAS tokens are now signed for the configured duration instead of a hardcoded 1 hour, so long jobs no longer fail with expired credentials.
- `ManifestFileCleanupTaskHandler` now handles Iceberg v2 delete manifests in addition to data manifests. Previously, `DROP TABLE PURGE` on a v2 table that had been updated via merge-on-read DML left position-delete files and their manifests as orphans in object storage; the cleanup task failed silently because `ManifestFiles.read()` rejects delete manifests.
- JDBC optimized location-overlap queries no longer include slash-only prefix terms such as `/` and `//`. Those were artifacts of stripping the URI scheme (e.g. `s3://bucket/path` → `//bucket/path`) and are not meaningful storage locations.

## [1.5.0]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -375,6 +375,10 @@ static PreparedQuery generateEntityTableExistQuery() {
* a path on one storage type may give a false positive for overlapping with another storage type.
* This should be combined with a check using `StorageLocation`.
*
* <p>Prefix equality terms that consist only of {@code /} characters are skipped. Those terms are
* artifacts of scheme stripping (e.g. {@code s3://bucket/path} becomes {@code //bucket/path} and
* would otherwise emit {@code /} and {@code //}), not meaningful storage locations.
*
* @param realmId A realm to search within
* @param schemaVersion The schema version of entities table to query
* @param catalogId A catalog entity to search within
Expand All @@ -395,14 +399,18 @@ public static PreparedQuery generateOverlapQuery(

for (String component : components) {
pathBuilder.append(component).append("/");
String prefix = pathBuilder.toString();
// Skip "/", "//", "///", ... produced from empty path segments around the scheme separator.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The /// prefix is probably valid. It's the root dir in file: URIs, right?

if (isSlashOnly(prefix)) {
continue;
}
conditions.add("location_without_scheme = ?");
parameters.add(pathBuilder.toString());
parameters.add(prefix);
Comment thread
vigneshio marked this conversation as resolved.
}

// Add LIKE condition to match children
conditions.add("location_without_scheme LIKE ?");
parameters.add(locationWithoutScheme + "%");

String locationClause = String.join(" OR ", conditions);
String clause = " WHERE realm_id = ? AND catalog_id = ? AND (" + locationClause + ")";

Expand All @@ -422,6 +430,20 @@ public static PreparedQuery generateOverlapQuery(
return new PreparedQuery(query.sql(), where.parameters());
}

/** True when {@code value} is non-empty and contains only {@code /} characters. */
@VisibleForTesting
static boolean isSlashOnly(String value) {
if (value == null || value.isEmpty()) {
return false;
}
for (int i = 0; i < value.length(); i++) {
if (value.charAt(i) != '/') {
return false;
}
}
return true;
}

static String getFullyQualifiedTableName(String tableName) {
// TODO: make schema name configurable.
return "POLARIS_SCHEMA." + tableName;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -349,50 +349,63 @@ void testGenerateUpdateQueryExtended_rejectsInvalidColumns() {

@Test
void testGenerateOverlapQuery() {
// s3://bucket/tmp/location/ → withoutScheme //bucket/tmp/location/
// Slash-only prefixes "/" and "//" from the scheme separator are not emitted.
assertEquals(
"SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code,"
+ " create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp,"
+ " properties, internal_properties, grant_records_version, location_without_scheme FROM"
+ " POLARIS_SCHEMA.ENTITIES WHERE realm_id = ? AND catalog_id = ? AND (location_without_scheme = ?"
+ " OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme = ? OR"
+ " location_without_scheme = ? OR location_without_scheme LIKE ?)",
+ " OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme LIKE ?)",
QueryGenerator.generateOverlapQuery("realmId", 2, -123, "s3://bucket/tmp/location/").sql());
Assertions.assertThatCollection(
QueryGenerator.generateOverlapQuery("realmId", 2, -123, "s3://bucket/tmp/location/")
.parameters())
.containsExactly(
"realmId",
-123L,
"/",
"//",
"//bucket/",
"//bucket/tmp/",
"//bucket/tmp/location/",
"//bucket/tmp/location/%");

// Absolute path becomes file:///tmp/location/ → withoutScheme ///tmp/location/
// Slash-only prefixes "/", "//", "///" are not emitted.
assertEquals(
"SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code,"
+ " create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp,"
+ " properties, internal_properties, grant_records_version, location_without_scheme FROM"
+ " POLARIS_SCHEMA.ENTITIES WHERE realm_id = ? AND catalog_id = ? AND (location_without_scheme = ? OR location_without_scheme = ?"
+ " OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme LIKE ?)",
+ " POLARIS_SCHEMA.ENTITIES WHERE realm_id = ? AND catalog_id = ? AND (location_without_scheme = ?"
+ " OR location_without_scheme = ? OR location_without_scheme LIKE ?)",
QueryGenerator.generateOverlapQuery("realmId", 2, -123, "/tmp/location/").sql());
Assertions.assertThatCollection(
QueryGenerator.generateOverlapQuery("realmId", 2, -123, "/tmp/location/").parameters())
.containsExactly(
"realmId", -123L, "/", "//", "///", "///tmp/", "///tmp/location/", "///tmp/location/%");
.containsExactly("realmId", -123L, "///tmp/", "///tmp/location/", "///tmp/location/%");

assertEquals(
"SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code,"
+ " create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp,"
+ " properties, internal_properties, grant_records_version, location_without_scheme"
+ " FROM POLARIS_SCHEMA.ENTITIES WHERE realm_id = ? AND catalog_id = ? AND (location_without_scheme = ?"
+ " OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme = ? OR location_without_scheme LIKE ?)",
+ " OR location_without_scheme = ? OR location_without_scheme LIKE ?)",
QueryGenerator.generateOverlapQuery("realmId", 2, -123, "s3://バケツ/\"loc.ation\"/").sql());
Assertions.assertThatCollection(
QueryGenerator.generateOverlapQuery("realmId", 2, -123, "s3://バケツ/\"loc.ation\"/")
.parameters())
.containsExactly(
"realmId", -123L, "/", "//", "//バケツ/", "//バケツ/\"loc.ation\"/", "//バケツ/\"loc.ation\"/%");
"realmId", -123L, "//バケツ/", "//バケツ/\"loc.ation\"/", "//バケツ/\"loc.ation\"/%");
}

@Test
void generateOverlapQueryDoesNotEmitSlashOnlyPrefixes() {

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: I don't see the coverage for a location without a trailing slash (e.g. s3://bucket/ns/t, IIUC, which is exactly where this and #5003 intersect. Worth a shared regression test once #5003 lands.

Assertions.assertThat(
QueryGenerator.generateOverlapQuery("realmId", 2, -123, "s3://bucket/ns/t/")
.parameters())
.doesNotContain("/", "//", "///");
Assertions.assertThat(
QueryGenerator.generateOverlapQuery("realmId", 2, -123, "file:///tmp/data/")
.parameters())
.filteredOn(p -> p instanceof String)
.noneMatch(p -> QueryGenerator.isSlashOnly((String) p));
}
}