diff --git a/CHANGELOG.md b/CHANGELOG.md index ec7a853939..3ddadba6df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,18 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti ### Highlights ### Upgrade notes +- Relational JDBC: a new schema version 5 makes the `events.catalog_id` column nullable. Events that + are not scoped to a catalog (principal, policy, rate-limiting, etc.) now persist `NULL` instead of + the placeholder string `__realm__` (which only ever existed in 1.6.0 release candidates). Fresh + bootstraps use schema v5 automatically. Existing deployments on schema v3/v4 keep working — the + server detects the older schema version and continues writing the legacy placeholder — but + operators are encouraged to upgrade manually, since Polaris has no automated schema migrations: + ```sql + ALTER TABLE polaris_schema.events ALTER COLUMN catalog_id DROP NOT NULL; + UPDATE polaris_schema.events SET catalog_id = NULL WHERE catalog_id = '__realm__'; + UPDATE polaris_schema.version SET version_value = 5 WHERE version_key = 'version'; + ``` + See the Relational JDBC metastore documentation for details. ### Breaking changes - Removed the `--schema-version` (`-v`) option from the admin tool's `bootstrap` command. New realms diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java index 365cb8262c..e89b7dc5eb 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java @@ -286,7 +286,7 @@ public void writeEvents(@NonNull List events) { QueryGenerator.generateInsertQuery( ModelEvent.ALL_COLUMNS, ModelEvent.TABLE_NAME, - ModelEvent.fromEvent(events.getFirst()) + ModelEvent.fromEvent(events.getFirst(), schemaVersion) .toMap(datasourceOperations.getDatabaseType()) .values() .stream() @@ -304,7 +304,7 @@ public void writeEvents(@NonNull List events) { QueryGenerator.generateInsertQuery( ModelEvent.ALL_COLUMNS, ModelEvent.TABLE_NAME, - ModelEvent.fromEvent(event) + ModelEvent.fromEvent(event, schemaVersion) .toMap(datasourceOperations.getDatabaseType()) .values() .stream() diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java index ff5f1111c5..0a3de44a25 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java @@ -43,6 +43,18 @@ public interface ModelEvent extends Converter { String RESOURCE_IDENTIFIER = "resource_identifier"; String ADDITIONAL_PROPERTIES = "additional_properties"; + /** + * Legacy sentinel stored in {@code events.catalog_id} for events that are not catalog-scoped on + * schema versions < 5, where the column is {@code NOT NULL}. Never exposed outside the JDBC + * layer: {@link #fromEvent(EventEntity, int)} substitutes it for a null catalog id on pre-v5 + * schemas, and {@link #fromResultSet} coalesces it back to null on read. + * + *

Because {@link #fromResultSet} cannot know which schema version a row was written under, the + * coalescing is unconditional, so this string remains reserved on all schema versions: a catalog + * literally named {@code __realm__} would have its events read back as realm-scoped. + */ + String LEGACY_REALM_SCOPED_CATALOG_ID = "__realm__"; + List ALL_COLUMNS = List.of( CATALOG_ID, @@ -62,7 +74,6 @@ public interface ModelEvent extends Converter { */ ModelEvent CONVERTER = ImmutableModelEvent.builder() - .catalogId("") .eventId("") .requestId("") .eventType("") @@ -73,8 +84,8 @@ public interface ModelEvent extends Converter { .additionalProperties("") .build(); - // catalog id - String getCatalogId(); + // catalog id, null for events that are not catalog-scoped + @Nullable String getCatalogId(); // event id String getEventId(); @@ -102,9 +113,10 @@ public interface ModelEvent extends Converter { @Override default EventEntity fromResultSet(ResultSet rs) throws SQLException { + String catalogId = rs.getString(CATALOG_ID); var modelEvent = ImmutableModelEvent.builder() - .catalogId(rs.getString(CATALOG_ID)) + .catalogId(LEGACY_REALM_SCOPED_CATALOG_ID.equals(catalogId) ? null : catalogId) .eventId(rs.getString(EVENT_ID)) .requestId(rs.getString(REQUEST_ID)) .eventType(rs.getString(EVENT_TYPE)) @@ -136,11 +148,16 @@ default Map toMap(DatabaseType databaseType) { return map; } - static ModelEvent fromEvent(EventEntity event) { + static ModelEvent fromEvent(EventEntity event, int schemaVersion) { if (event == null) return null; + String catalogId = event.getCatalogId(); + if (schemaVersion < 5 && catalogId == null) { + // Schema versions < 5 declare events.catalog_id NOT NULL; store the legacy sentinel there. + catalogId = LEGACY_REALM_SCOPED_CATALOG_ID; + } return ImmutableModelEvent.builder() - .catalogId(event.getCatalogId()) + .catalogId(catalogId) .eventId(event.getId()) .requestId(event.getRequestId()) .eventType(event.getEventType()) diff --git a/persistence/relational-jdbc/src/main/resources/cockroachdb/schema-v5.sql b/persistence/relational-jdbc/src/main/resources/cockroachdb/schema-v5.sql index 57a00efb6b..3450329d5b 100644 --- a/persistence/relational-jdbc/src/main/resources/cockroachdb/schema-v5.sql +++ b/persistence/relational-jdbc/src/main/resources/cockroachdb/schema-v5.sql @@ -21,6 +21,7 @@ -- Changes from v4: -- * Removed the `idempotency_records` table (the durable idempotency store it -- backed was never wired into any request path and has been removed) +-- * `events.catalog_id` is nullable; events that are not catalog-scoped store NULL (issue #4674) -- Features: -- * Uses INT4 explicitly for all integer columns (required for CockroachDB JDBC driver) -- * Includes all tables: version, entities, grant_records, principal_authentication_data, @@ -131,7 +132,7 @@ CREATE INDEX IF NOT EXISTS idx_policy_mapping_record ON policy_mapping_record (r CREATE TABLE IF NOT EXISTS events ( realm_id TEXT NOT NULL, - catalog_id TEXT NOT NULL, + catalog_id TEXT, event_id TEXT NOT NULL, request_id TEXT, event_type TEXT NOT NULL, diff --git a/persistence/relational-jdbc/src/main/resources/h2/schema-v5.sql b/persistence/relational-jdbc/src/main/resources/h2/schema-v5.sql index 2933ab7c20..de7fb2484b 100644 --- a/persistence/relational-jdbc/src/main/resources/h2/schema-v5.sql +++ b/persistence/relational-jdbc/src/main/resources/h2/schema-v5.sql @@ -20,6 +20,7 @@ -- Changes from v4: -- * Removed the `idempotency_records` table (the durable idempotency store it -- backed was never wired into any request path and has been removed) +-- * `events.catalog_id` is nullable; events that are not catalog-scoped store NULL (issue #4674) CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; SET SCHEMA POLARIS_SCHEMA; @@ -130,7 +131,7 @@ CREATE INDEX IF NOT EXISTS idx_policy_mapping_record ON policy_mapping_record (r CREATE TABLE IF NOT EXISTS events ( realm_id TEXT NOT NULL, - catalog_id TEXT NOT NULL, + catalog_id TEXT, event_id TEXT NOT NULL, request_id TEXT, event_type TEXT NOT NULL, diff --git a/persistence/relational-jdbc/src/main/resources/postgres/schema-v5.sql b/persistence/relational-jdbc/src/main/resources/postgres/schema-v5.sql index b2dc63e65d..7b4e7a0719 100644 --- a/persistence/relational-jdbc/src/main/resources/postgres/schema-v5.sql +++ b/persistence/relational-jdbc/src/main/resources/postgres/schema-v5.sql @@ -19,6 +19,7 @@ -- Changes from v4: -- * Removed the `idempotency_records` table (the durable idempotency store it -- backed was never wired into any request path and has been removed) +-- * `events.catalog_id` is nullable; events that are not catalog-scoped store NULL (issue #4674) CREATE SCHEMA IF NOT EXISTS POLARIS_SCHEMA; SET search_path TO POLARIS_SCHEMA; @@ -130,7 +131,7 @@ CREATE INDEX IF NOT EXISTS idx_policy_mapping_record ON policy_mapping_record (r CREATE TABLE IF NOT EXISTS events ( realm_id TEXT NOT NULL, - catalog_id TEXT NOT NULL, + catalog_id TEXT, event_id TEXT NOT NULL, request_id TEXT, event_type TEXT NOT NULL, diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperationsTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperationsTest.java index 29bbd6c49b..02b1382cb5 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperationsTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperationsTest.java @@ -110,7 +110,7 @@ void executeBatchUpdate_success() throws Exception { ImmutableModelEvent.builder() .resourceType(EventEntity.ResourceType.CATALOG) .resourceIdentifier("catalog_" + i) - .catalogId("catalog_" + i) + .catalogId(i % 2 == 0 ? "catalog_" + i : null) .eventId("event_" + i) .requestId("request_" + i) .eventType("event_type1") diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcEventsPersistenceTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcEventsPersistenceTest.java new file mode 100644 index 0000000000..aae803aea5 --- /dev/null +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcEventsPersistenceTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.InputStream; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.List; +import java.util.UUID; +import java.util.stream.Stream; +import javax.sql.DataSource; +import org.apache.polaris.core.PolarisDefaultDiagServiceImpl; +import org.apache.polaris.core.entity.EventEntity; +import org.apache.polaris.core.persistence.PrincipalSecretsGenerator; +import org.apache.polaris.persistence.relational.jdbc.models.ModelEvent; +import org.h2.jdbcx.JdbcConnectionPool; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Tests event persistence across schema versions: schema versions before v5 declare + * events.catalog_id NOT NULL, so realm-scoped events (null catalog id) must be stored with the + * legacy sentinel; from v5 on they are stored as SQL NULL. + */ +class JdbcEventsPersistenceTest { + + static Stream schemaVersions() { + // The events table exists from schema v3 on. + return SchemaVersions.discoverAsStream(DatabaseType.H2).filter(version -> version >= 3); + } + + @ParameterizedTest + @MethodSource("schemaVersions") + void writeEventsStoresNullCatalogIdPerSchemaVersion(int schemaVersion) throws Exception { + DataSource dataSource = + JdbcConnectionPool.create( + "jdbc:h2:mem:test_events_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1", "sa", ""); + DatasourceOperations datasourceOperations = + new DatasourceOperations( + dataSource, SimpleRelationalJdbcConfiguration.forDatabaseType(DatabaseType.H2)); + try (InputStream schemaStream = DatabaseType.H2.openInitScriptResource(schemaVersion)) { + datasourceOperations.executeScript(schemaStream); + } + JdbcBasePersistenceImpl persistence = + new JdbcBasePersistenceImpl( + new PolarisDefaultDiagServiceImpl(), + datasourceOperations, + PrincipalSecretsGenerator.RANDOM_SECRETS, + "TEST_REALM", + schemaVersion); + + EventEntity realmScopedEvent = + new EventEntity( + null, + "realm-event", + null, + "CREATE_PRINCIPAL", + 1234L, + "test-user", + EventEntity.ResourceType.REALM, + "principal-1"); + EventEntity catalogScopedEvent = + new EventEntity( + "catalog-1", + "catalog-event", + "req-1", + "CREATE_TABLE", + 1234L, + "test-user", + EventEntity.ResourceType.TABLE, + "table-1"); + + persistence.writeEvents(List.of(realmScopedEvent, catalogScopedEvent)); + + if (schemaVersion < 5) { + assertEquals( + ModelEvent.LEGACY_REALM_SCOPED_CATALOG_ID, + readStoredCatalogId(dataSource, "realm-event")); + } else { + assertNull(readStoredCatalogId(dataSource, "realm-event")); + } + assertEquals("catalog-1", readStoredCatalogId(dataSource, "catalog-event")); + } + + private static String readStoredCatalogId(DataSource dataSource, String eventId) + throws Exception { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = + connection.prepareStatement( + "SELECT catalog_id FROM POLARIS_SCHEMA.EVENTS WHERE event_id = ?")) { + statement.setString(1, eventId); + try (ResultSet resultSet = statement.executeQuery()) { + assertTrue(resultSet.next()); + return resultSet.getString(1); + } + } + } +} diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEventTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEventTest.java index d39e245621..dd4a70a13e 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEventTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEventTest.java @@ -30,6 +30,7 @@ import static org.apache.polaris.persistence.relational.jdbc.models.ModelEvent.TIMESTAMP_MS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -54,6 +55,7 @@ public class ModelEventTest { private static final String TEST_RESOURCE_IDENTIFIER = "test-table"; private static final String EMPTY_JSON = "{}"; private static final String TEST_JSON = "{\"key\":\"value\"}"; + private static final int LATEST_SCHEMA_VERSION = DatabaseType.H2.getLatestSchemaVersion(); @Test public void testFromResultSet() throws SQLException { @@ -84,6 +86,78 @@ public void testFromResultSet() throws SQLException { assertEquals(EMPTY_JSON, result.getAdditionalProperties()); } + @Test + public void testFromResultSetCoalescesLegacyRealmScopedCatalogId() throws SQLException { + // Arrange + ResultSet mockResultSet = mock(ResultSet.class); + when(mockResultSet.getString(CATALOG_ID)).thenReturn(ModelEvent.LEGACY_REALM_SCOPED_CATALOG_ID); + when(mockResultSet.getString(EVENT_ID)).thenReturn(TEST_EVENT_ID); + when(mockResultSet.getString(REQUEST_ID)).thenReturn(TEST_REQUEST_ID); + when(mockResultSet.getString(EVENT_TYPE)).thenReturn(TEST_EVENT_TYPE); + when(mockResultSet.getLong(TIMESTAMP_MS)).thenReturn(TEST_TIMESTAMP_MS); + when(mockResultSet.getString(PRINCIPAL_NAME)).thenReturn(TEST_USER); + when(mockResultSet.getString(RESOURCE_TYPE)).thenReturn(TEST_RESOURCE_TYPE_STRING); + when(mockResultSet.getString(RESOURCE_IDENTIFIER)).thenReturn(TEST_RESOURCE_IDENTIFIER); + when(mockResultSet.getString(ADDITIONAL_PROPERTIES)).thenReturn(EMPTY_JSON); + + // Act + EventEntity result = ModelEvent.CONVERTER.fromResultSet(mockResultSet); + + // Assert + assertNull(result.getCatalogId()); + assertEquals(TEST_EVENT_ID, result.getId()); + } + + @Test + public void testRoundTripWithNullCatalogId() { + // Arrange + EventEntity polarisEvent = + new EventEntity( + null, + TEST_EVENT_ID, + TEST_REQUEST_ID, + TEST_EVENT_TYPE, + TEST_TIMESTAMP_MS, + TEST_USER, + TEST_RESOURCE_TYPE, + TEST_RESOURCE_IDENTIFIER); + polarisEvent.setAdditionalProperties(TEST_JSON); + + // Act + ModelEvent modelEvent = ModelEvent.fromEvent(polarisEvent, LATEST_SCHEMA_VERSION); + Map resultMap = modelEvent.toMap(DatabaseType.H2); + EventEntity result = ModelEvent.toEvent(modelEvent); + + // Assert + assertNull(modelEvent.getCatalogId()); + assertTrue(resultMap.containsKey(CATALOG_ID)); + assertNull(resultMap.get(CATALOG_ID)); + assertNull(result.getCatalogId()); + assertEquals(TEST_EVENT_ID, result.getId()); + assertEquals(TEST_JSON, result.getAdditionalProperties()); + } + + @Test + public void testFromEventStoresLegacySentinelForNullCatalogIdOnPreV5Schema() { + // Arrange + EventEntity polarisEvent = + new EventEntity( + null, + TEST_EVENT_ID, + TEST_REQUEST_ID, + TEST_EVENT_TYPE, + TEST_TIMESTAMP_MS, + TEST_USER, + TEST_RESOURCE_TYPE, + TEST_RESOURCE_IDENTIFIER); + + // Act + ModelEvent modelEvent = ModelEvent.fromEvent(polarisEvent, 4); + + // Assert + assertEquals(ModelEvent.LEGACY_REALM_SCOPED_CATALOG_ID, modelEvent.getCatalogId()); + } + @Test public void testToMapWithH2DatabaseType() { // Arrange @@ -153,7 +227,7 @@ public void testToMapWithPostgresType() { @Test public void testFromEventWithNullInput() { // Act - ModelEvent result = ModelEvent.fromEvent(null); + ModelEvent result = ModelEvent.fromEvent(null, LATEST_SCHEMA_VERSION); // Assert assertNull(result); @@ -175,7 +249,7 @@ public void testFromEvent() { polarisEvent.setAdditionalProperties(TEST_JSON); // Act - ModelEvent result = ModelEvent.fromEvent(polarisEvent); + ModelEvent result = ModelEvent.fromEvent(polarisEvent, LATEST_SCHEMA_VERSION); // Assert assertEquals(TEST_CATALOG_ID, result.getCatalogId()); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/EventEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/EventEntity.java index cefa754ee6..4f693c6ad5 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/EventEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/EventEntity.java @@ -29,31 +29,15 @@ public class EventEntity { public static final String EMPTY_MAP_STRING = "{}"; - /** - * Sentinel stored in {@link #catalogId} for events that are not scoped to a specific catalog - * (e.g., principal, principal-role, or other realm-level operations). Downstream consumers - * querying by catalog must filter this sentinel out (or coalesce it to {@code NULL}). - * - *

This is a transitional placeholder: {@code catalogId} is currently {@code NOT NULL} in the - * persistence schema, so realm-scoped events cannot store {@code null} directly. A future schema - * migration may relax that constraint and let realm-scoped events use {@code NULL} instead. - * - *

TODO: Remove this sentinel once the {@code catalogId} column is made nullable. Realm-scoped - * events should store {@code NULL}, not a magic string, so downstream queries can use {@code IS - * NULL} instead of filtering out a reserved value. - */ - public static final String REALM_SCOPED = "__realm__"; - // to serialize/deserialize properties // TODO: Look into using the CDI-managed `ObjectMapper` object private static final ObjectMapper MAPPER = JsonMapper.shared(); /** - * Identifier of the catalog this event was scoped to, or {@link #REALM_SCOPED} for events that - * are not catalog-scoped (e.g., principal/realm-level operations). See {@link #REALM_SCOPED} for - * the sentinel convention and planned schema evolution. + * Identifier of the catalog this event was scoped to, or {@code null} for events that are not + * catalog-scoped (e.g., principal/realm-level operations). */ - private final String catalogId; + @Nullable private final String catalogId; // event id private final String id; @@ -79,6 +63,7 @@ public class EventEntity { // Additional parameters that were not earlier recorded private String additionalProperties; + @Nullable public String getCatalogId() { return catalogId; } @@ -118,7 +103,7 @@ public String getAdditionalProperties() { } public EventEntity( - String catalogId, + @Nullable String catalogId, String id, @Nullable String requestId, String eventType, diff --git a/runtime/service/src/main/java/org/apache/polaris/service/events/listeners/PolarisPersistenceEventListener.java b/runtime/service/src/main/java/org/apache/polaris/service/events/listeners/PolarisPersistenceEventListener.java index d8d21a5d06..46720ab35c 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/events/listeners/PolarisPersistenceEventListener.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/events/listeners/PolarisPersistenceEventListener.java @@ -37,6 +37,7 @@ import org.apache.polaris.service.events.EventAttributes; import org.apache.polaris.service.events.PolarisEvent; import org.apache.polaris.service.events.PolarisEventType; +import org.jspecify.annotations.Nullable; public abstract class PolarisPersistenceEventListener implements PolarisEventListener { @@ -67,8 +68,9 @@ public void onEvent(PolarisEvent event) { processEvent(event.metadata().realmId(), polarisEvent); } + @Nullable private static String resolveCatalogName(PolarisEvent event) { - return event.attributes().get(EventAttributes.CATALOG_NAME).orElse(EventEntity.REALM_SCOPED); + return event.attributes().get(EventAttributes.CATALOG_NAME).orElse(null); } /** @@ -98,7 +100,7 @@ private static ResourceType resolveResourceType(PolarisEventType eventType) { } private static String resolveResourceIdentifier( - PolarisEvent event, ResourceType resourceType, String catalogName) { + PolarisEvent event, ResourceType resourceType, @Nullable String catalogName) { return switch (resourceType) { case TABLE -> resolveTableResourceIdentifier(event, catalogName); case VIEW -> resolveViewResourceIdentifier(event, catalogName); @@ -108,7 +110,8 @@ private static String resolveResourceIdentifier( }; } - private static String resolveTableResourceIdentifier(PolarisEvent event, String catalogName) { + private static String resolveTableResourceIdentifier( + PolarisEvent event, @Nullable String catalogName) { EventAttributeMap attributes = event.attributes(); Optional identifierFromTableAttribute = @@ -148,7 +151,8 @@ private static String resolveTableResourceIdentifier(PolarisEvent event, String .orElseGet(() -> fallbackResourceIdentifier(event, catalogName)); } - private static String resolveViewResourceIdentifier(PolarisEvent event, String catalogName) { + private static String resolveViewResourceIdentifier( + PolarisEvent event, @Nullable String catalogName) { EventAttributeMap attributes = event.attributes(); return attributes .get(EventAttributes.VIEW_IDENTIFIER) @@ -212,7 +216,8 @@ private static Optional resolveRegisterTableIdentifier(EventAttributeMap .map(tableName -> TableIdentifier.of(namespace, tableName).toString())); } - private static String resolveNamespaceResourceIdentifier(PolarisEvent event, String catalogName) { + private static String resolveNamespaceResourceIdentifier( + PolarisEvent event, @Nullable String catalogName) { EventAttributeMap attributes = event.attributes(); return attributes .get(EventAttributes.NAMESPACE) @@ -222,7 +227,8 @@ private static String resolveNamespaceResourceIdentifier(PolarisEvent event, Str .orElseGet(() -> fallbackResourceIdentifier(event, catalogName)); } - private static String resolveCatalogResourceIdentifier(PolarisEvent event, String catalogName) { + private static String resolveCatalogResourceIdentifier( + PolarisEvent event, @Nullable String catalogName) { return event .attributes() .get(EventAttributes.CATALOG_NAME) @@ -233,11 +239,9 @@ private static String resolveRealmResourceIdentifier(PolarisEvent event) { return event.type().name(); } - private static String fallbackResourceIdentifier(PolarisEvent event, String catalogName) { - if (!EventEntity.REALM_SCOPED.equals(catalogName)) { - return catalogName; - } - return event.type().name(); + private static String fallbackResourceIdentifier( + PolarisEvent event, @Nullable String catalogName) { + return catalogName != null ? catalogName : event.type().name(); } private static Map buildAdditionalProperties(PolarisEvent event) { diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/PolarisPersistenceEventListenerTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/PolarisPersistenceEventListenerTest.java index 0e21e9f5f1..9a11c59d17 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/PolarisPersistenceEventListenerTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/PolarisPersistenceEventListenerTest.java @@ -193,7 +193,7 @@ void shouldFallbackWhenNoCatalogOrResourceAttributesExist() { PolarisEventType.BEFORE_LIMIT_REQUEST_RATE, metadata(), new EventAttributeMap())); EventEntity persisted = listener.persistedEvent(PolarisEventType.BEFORE_LIMIT_REQUEST_RATE); - assertThat(persisted.getCatalogId()).isEqualTo(EventEntity.REALM_SCOPED); + assertThat(persisted.getCatalogId()).isNull(); assertThat(persisted.getResourceType()).isEqualTo(EventEntity.ResourceType.REALM); assertThat(persisted.getResourceIdentifier()) .isEqualTo(PolarisEventType.BEFORE_LIMIT_REQUEST_RATE.name()); diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java index 4fbca555db..16d8842d02 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java @@ -44,6 +44,8 @@ import java.sql.Statement; import java.time.Duration; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import javax.sql.DataSource; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; @@ -151,6 +153,10 @@ void testCreateCatalogAndTable() throws IOException { .create(); } + try (Response response = managementApi.request("v1/principals").get()) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + } + String query = "SELECT * FROM polaris_schema.events WHERE realm_id = '" + realm @@ -172,7 +178,15 @@ void testCreateCatalogAndTable() throws IOException { } return e.build(); }, - e -> e.size() >= 2); + e -> + e.stream() + .map(EventEntity::getEventType) + .collect(Collectors.toSet()) + .containsAll( + Set.of( + "AFTER_CREATE_CATALOG", + "AFTER_CREATE_TABLE", + "AFTER_LIST_PRINCIPALS"))); EventEntity e1 = events.stream() @@ -214,5 +228,14 @@ void testCreateCatalogAndTable() throws IOException { OPEN_TELEMETRY_TRACE_ID_KEY, value -> assertThat(value).matches("[0-9a-f]{32}")) .hasEntrySatisfying( OPEN_TELEMETRY_SPAN_ID_KEY, value -> assertThat(value).matches("[0-9a-f]{16}")); + + EventEntity e3 = + events.stream() + .filter(e -> e.getEventType().equals("AFTER_LIST_PRINCIPALS")) + .findFirst() + .orElseThrow(); + assertThat(e3.getCatalogId()).isNull(); + assertThat(e3.getResourceType()).isEqualTo(EventEntity.ResourceType.REALM); + assertThat(e3.getResourceIdentifier()).isEqualTo("AFTER_LIST_PRINCIPALS"); } } diff --git a/site/content/in-dev/unreleased/metastores/relational-jdbc.md b/site/content/in-dev/unreleased/metastores/relational-jdbc.md index 037cb0fafe..bae952e875 100644 --- a/site/content/in-dev/unreleased/metastores/relational-jdbc.md +++ b/site/content/in-dev/unreleased/metastores/relational-jdbc.md @@ -101,3 +101,28 @@ java \ ``` For more details on the bootstrap command and other administrative operations, see the [Admin Tool]({{% ref "../admin-tool" %}}) documentation. + +## Schema upgrades + +Polaris does not run automated schema migrations. Bootstrapping applies a full `schema-vN.sql` +script and records the schema version in the `polaris_schema.version` table; upgrading an existing +database to a newer schema version is a manual, operator-driven step. + +### Upgrading to schema v5 + +Schema v5 makes the `events.catalog_id` column nullable: events that are not scoped to a catalog +(principal, policy, rate-limiting, etc.) store `NULL` instead of the legacy placeholder string +`__realm__` that pre-v5 schemas required (the placeholder only ever existed in 1.6.0 release +candidates). + +Until you upgrade, the server keeps working: it detects a schema version below 5 from the +`polaris_schema.version` table at startup and continues writing the legacy placeholder for events +that are not catalog-scoped. To upgrade an existing v3/v4 database, run the following one-time SQL +(adjust the `ALTER` syntax to your database if needed — the statements below work on PostgreSQL, +CockroachDB, and H2), then restart Polaris: + +```sql +ALTER TABLE polaris_schema.events ALTER COLUMN catalog_id DROP NOT NULL; +UPDATE polaris_schema.events SET catalog_id = NULL WHERE catalog_id = '__realm__'; +UPDATE polaris_schema.version SET version_value = 5 WHERE version_key = 'version'; +```