From 9070e24511fbb2c26d478be758464b60eaa53461 Mon Sep 17 00:00:00 2001 From: Yufei Date: Mon, 6 Jul 2026 10:44:07 -0700 Subject: [PATCH 1/3] Add Polaris-managed JDBC datasource configuration Allow the relational JDBC backend to create and own a Hikari datasource from polaris.persistence.relational.jdbc.* settings when jdbc-url is provided, while preserving the existing Quarkus datasource fallback when it is absent. Cover the new path with unit and integration tests for managed datasource creation, independent datasource creation from separate configs, CockroachDB configuration, production-readiness checks, and loading a JDBC driver from a runtime-provided jar. Document the managed datasource properties and update distribution license metadata for the added runtime dependencies. --- gradle/libs.versions.toml | 1 + persistence/relational-jdbc/build.gradle.kts | 1 + .../relational/jdbc/DatasourceOperations.java | 26 +- .../jdbc/JdbcDataSourceFactory.java | 54 ++++ .../jdbc/JdbcMetaStoreManagerFactory.java | 31 +- .../jdbc/RelationalJdbcConfiguration.java | 35 +++ ...lationalJdbcProductionReadinessChecks.java | 7 +- .../jdbc/JdbcDataSourceFactoryTest.java | 278 ++++++++++++++++++ .../jdbc/JdbcMetaStoreManagerFactoryTest.java | 124 ++++++++ ...onalJdbcProductionReadinessChecksTest.java | 20 ++ .../TestingRelationalJdbcConfiguration.java | 160 ++++++++++ runtime/admin/distribution/LICENSE | 10 + .../QuarkusRelationalJdbcConfiguration.java | 25 +- .../src/main/resources/application.properties | 10 + runtime/server/distribution/LICENSE | 10 + .../jdbc/PolarisManagedCockroachJdbcIT.java | 96 ++++++ .../PolarisManagedJdbcIntegrationTest.java | 96 ++++++ ...oachRelationalJdbcLifeCycleManagement.java | 42 +-- ...rye-polaris_persistence_relational_jdbc.md | 7 + .../_index.md | 11 +- .../unreleased/metastores/relational-jdbc.md | 70 ++++- 21 files changed, 1074 insertions(+), 40 deletions(-) create mode 100644 persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactory.java create mode 100644 persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactoryTest.java create mode 100644 persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactoryTest.java create mode 100644 persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/TestingRelationalJdbcConfiguration.java create mode 100644 runtime/service/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/PolarisManagedCockroachJdbcIT.java create mode 100644 runtime/service/src/test/java/org/apache/polaris/persistence/relational/jdbc/PolarisManagedJdbcIntegrationTest.java diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b03f3d5d7f..7624757148 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -68,6 +68,7 @@ hadoop-client-api = { module = "org.apache.hadoop:hadoop-client-api", version.re hadoop-client-runtime = { module = "org.apache.hadoop:hadoop-client-runtime", version.ref = "hadoop" } hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop" } hawkular-agent-prometheus-scraper = { module = "org.hawkular.agent:prometheus-scraper", version = "0.23.0.Final" } +hikari-cp = { module = "com.zaxxer:HikariCP", version = "7.1.0" } hive-metastore = { module = "org.apache.hive:hive-metastore", version = "3.1.3" } iceberg-bom = { module = "org.apache.iceberg:iceberg-bom", version.ref = "iceberg" } immutables-builder = { module = "org.immutables:builder", version.ref = "immutables" } diff --git a/persistence/relational-jdbc/build.gradle.kts b/persistence/relational-jdbc/build.gradle.kts index d0a3c15cfc..4aa47f68a2 100644 --- a/persistence/relational-jdbc/build.gradle.kts +++ b/persistence/relational-jdbc/build.gradle.kts @@ -26,6 +26,7 @@ dependencies { implementation(project(":polaris-core")) implementation(libs.slf4j.api) implementation(libs.guava) + implementation(libs.hikari.cp) implementation(platform(libs.jackson3.bom)) implementation("tools.jackson.core:jackson-core") diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java index 372ccfe1ca..ae4a42c277 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java @@ -49,7 +49,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class DatasourceOperations { +public class DatasourceOperations implements AutoCloseable { private static final Logger LOGGER = LoggerFactory.getLogger(DatasourceOperations.class); @@ -69,14 +69,23 @@ public class DatasourceOperations { private final DataSource datasource; private final RelationalJdbcConfiguration relationalJdbcConfiguration; + private final boolean closeDataSourceOnClose; private final DatabaseType databaseType; private static final Random random = new Random(); public DatasourceOperations( DataSource datasource, RelationalJdbcConfiguration relationalJdbcConfiguration) { + this(datasource, relationalJdbcConfiguration, false); + } + + DatasourceOperations( + DataSource datasource, + RelationalJdbcConfiguration relationalJdbcConfiguration, + boolean closeDataSourceOnClose) { this.datasource = datasource; this.relationalJdbcConfiguration = relationalJdbcConfiguration; + this.closeDataSourceOnClose = closeDataSourceOnClose; try (Connection connection = this.datasource.getConnection()) { // Get explicitly configured database type, if any DatabaseType configuredType = @@ -98,6 +107,21 @@ DatabaseType getDatabaseType() { return databaseType; } + boolean ownsDataSource() { + return closeDataSourceOnClose; + } + + @Override + public void close() { + if (closeDataSourceOnClose && datasource instanceof AutoCloseable closeable) { + try { + closeable.close(); + } catch (Exception e) { + throw new RuntimeException("Failed to close datasource", e); + } + } + } + /** * Execute SQL script and close the associated input stream * diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactory.java new file mode 100644 index 0000000000..a5c0c4bc14 --- /dev/null +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactory.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.util.Optional; +import javax.sql.DataSource; + +final class JdbcDataSourceFactory { + + private static final String POOL_NAME = "polaris-relational-jdbc"; + + private JdbcDataSourceFactory() {} + + static Optional create(RelationalJdbcConfiguration configuration) { + Optional jdbcUrl = nonBlank(configuration.jdbcUrl()); + if (jdbcUrl.isEmpty()) { + return Optional.empty(); + } + + HikariConfig hikariConfig = new HikariConfig(); + hikariConfig.setPoolName(POOL_NAME); + hikariConfig.setJdbcUrl(jdbcUrl.get()); + nonBlank(configuration.driver()).ifPresent(hikariConfig::setDriverClassName); + nonBlank(configuration.username()).ifPresent(hikariConfig::setUsername); + configuration.password().ifPresent(hikariConfig::setPassword); + configuration.maximumPoolSize().ifPresent(hikariConfig::setMaximumPoolSize); + configuration.minimumIdle().ifPresent(hikariConfig::setMinimumIdle); + configuration.connectionTimeoutInMs().ifPresent(hikariConfig::setConnectionTimeout); + + return Optional.of(new HikariDataSource(hikariConfig)); + } + + private static Optional nonBlank(Optional value) { + return value.map(String::trim).filter(s -> !s.isEmpty()); + } +} diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index e88aac882a..8eda1753cf 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -22,6 +22,7 @@ import io.smallrye.common.annotation.Identifier; import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Disposes; import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.Produces; import jakarta.inject.Inject; @@ -95,7 +96,35 @@ protected JdbcMetaStoreManagerFactory() {} @ApplicationScoped static DatasourceOperations produceDatasourceOperations( Instance dataSource, RelationalJdbcConfiguration relationalJdbcConfiguration) { - return new DatasourceOperations(dataSource.get(), relationalJdbcConfiguration); + Optional polarisDataSource = + JdbcDataSourceFactory.create(relationalJdbcConfiguration); + return polarisDataSource + .map(ds -> createOwnedDatasourceOperations(ds, relationalJdbcConfiguration)) + .orElseGet(() -> new DatasourceOperations(dataSource.get(), relationalJdbcConfiguration)); + } + + static void closeDatasourceOperations(@Disposes DatasourceOperations datasourceOperations) { + datasourceOperations.close(); + } + + private static DatasourceOperations createOwnedDatasourceOperations( + DataSource dataSource, RelationalJdbcConfiguration relationalJdbcConfiguration) { + try { + return new DatasourceOperations(dataSource, relationalJdbcConfiguration, true); + } catch (RuntimeException e) { + closeDataSource(dataSource, e); + throw e; + } + } + + private static void closeDataSource(DataSource dataSource, RuntimeException originalException) { + if (dataSource instanceof AutoCloseable closeable) { + try { + closeable.close(); + } catch (Exception closeException) { + originalException.addSuppressed(closeException); + } + } } protected PrincipalSecretsGenerator secretsGenerator( diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java index adfcae5ab1..652f77f7b9 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java @@ -35,4 +35,39 @@ public interface RelationalJdbcConfiguration { * the JDBC connection metadata. Supported values: "postgresql", "cockroachdb", "h2" */ Optional databaseType(); + + /** JDBC URL used when Polaris creates and owns the relational JDBC datasource. */ + default Optional jdbcUrl() { + return Optional.empty(); + } + + /** JDBC driver class name used when Polaris creates and owns the datasource. */ + default Optional driver() { + return Optional.empty(); + } + + /** Database username used when Polaris creates and owns the datasource. */ + default Optional username() { + return Optional.empty(); + } + + /** Database password used when Polaris creates and owns the datasource. */ + default Optional password() { + return Optional.empty(); + } + + /** Maximum number of connections in the Polaris-managed datasource pool. */ + default Optional maximumPoolSize() { + return Optional.empty(); + } + + /** Minimum number of idle connections in the Polaris-managed datasource pool. */ + default Optional minimumIdle() { + return Optional.empty(); + } + + /** Maximum time to wait for a connection from the Polaris-managed datasource pool. */ + default Optional connectionTimeoutInMs() { + return Optional.empty(); + } } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecks.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecks.java index 62c34975d9..3ebd79cf52 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecks.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecks.java @@ -36,11 +36,14 @@ public ProductionReadinessCheck checkRelationalJdbc( return ProductionReadinessCheck.OK; } - if (datasourceOperations.get().getDatabaseType().equals(DatabaseType.H2)) { + DatasourceOperations operations = datasourceOperations.get(); + if (operations.getDatabaseType().equals(DatabaseType.H2)) { return ProductionReadinessCheck.of( ProductionReadinessCheck.Error.of( "The current persistence (jdbc:h2) is intended for tests only.", - "quarkus.datasource.jdbc.url")); + operations.ownsDataSource() + ? "polaris.persistence.relational.jdbc.jdbc-url" + : "quarkus.datasource.jdbc.url")); } return ProductionReadinessCheck.OK; } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactoryTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactoryTest.java new file mode 100644 index 0000000000..8808faa791 --- /dev/null +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactoryTest.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.zaxxer.hikari.HikariDataSource; +import java.io.File; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import javax.sql.DataSource; +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class JdbcDataSourceFactoryTest { + + @Test + void createReturnsEmptyWhenJdbcUrlIsAbsent() { + RelationalJdbcConfiguration configuration = + TestingRelationalJdbcConfiguration.builder().build(); + + assertThat(JdbcDataSourceFactory.create(configuration)).isEmpty(); + } + + @Test + void createBuildsHikariDataSourceFromRuntimeConfig() throws Exception { + String jdbcUrl = "jdbc:h2:mem:runtime_config_poc;DB_CLOSE_DELAY=-1"; + RelationalJdbcConfiguration configuration = + TestingRelationalJdbcConfiguration.builder() + .jdbcUrl(jdbcUrl) + .driver("org.h2.Driver") + .username("sa") + .password("") + .maximumPoolSize(2) + .minimumIdle(1) + .connectionTimeoutInMs(1000L) + .build(); + + DataSource dataSource = JdbcDataSourceFactory.create(configuration).orElseThrow(); + try { + assertThat(dataSource).isInstanceOf(HikariDataSource.class); + HikariDataSource hikariDataSource = (HikariDataSource) dataSource; + assertThat(hikariDataSource.getJdbcUrl()).isEqualTo(jdbcUrl); + assertThat(hikariDataSource.getMaximumPoolSize()).isEqualTo(2); + assertThat(hikariDataSource.getMinimumIdle()).isEqualTo(1); + assertThat(hikariDataSource.getConnectionTimeout()).isEqualTo(1000L); + + try (Connection connection = dataSource.getConnection()) { + assertThat(connection.getMetaData().getDatabaseProductName()).isEqualTo("H2"); + } + } finally { + ((HikariDataSource) dataSource).close(); + } + } + + @Test + void createBuildsIndependentDataSourcesForDifferentRealmConfigurations() throws Exception { + HikariDataSource engineeringDataSource = + createDataSource("jdbc:h2:mem:engineering_realm_poc;DB_CLOSE_DELAY=-1"); + HikariDataSource financeDataSource = + createDataSource("jdbc:h2:mem:finance_realm_poc;DB_CLOSE_DELAY=-1"); + + try { + writeRealmMarker(engineeringDataSource, "engineering"); + writeRealmMarker(financeDataSource, "finance"); + + assertThat(readRealmMarker(engineeringDataSource)).isEqualTo("engineering"); + assertThat(readRealmMarker(financeDataSource)).isEqualTo("finance"); + } finally { + engineeringDataSource.close(); + financeDataSource.close(); + } + } + + @Test + void createUsesJdbcDriverLoadedFromRuntimeJar(@TempDir Path tempDir) throws Exception { + String driverClassName = "runtime.jdbc.RuntimeLoadedJdbcDriver"; + Path driverJar = createRuntimeDriverJar(tempDir, driverClassName); + + assertThatThrownBy( + () -> + Class.forName( + driverClassName, false, JdbcDataSourceFactoryTest.class.getClassLoader())) + .isInstanceOf(ClassNotFoundException.class); + + ClassLoader previousContextClassLoader = Thread.currentThread().getContextClassLoader(); + try (URLClassLoader runtimeClassLoader = + new URLClassLoader(new URL[] {driverJar.toUri().toURL()}, previousContextClassLoader)) { + Thread.currentThread().setContextClassLoader(runtimeClassLoader); + HikariDataSource dataSource = + createDataSource( + "jdbc:polaris-runtime-h2:mem:runtime_loaded_driver_poc;DB_CLOSE_DELAY=-1", + driverClassName); + + try { + writeRealmMarker(dataSource, "runtime-loaded-driver"); + + assertThat(readRealmMarker(dataSource)).isEqualTo("runtime-loaded-driver"); + } finally { + dataSource.close(); + } + } finally { + Thread.currentThread().setContextClassLoader(previousContextClassLoader); + } + } + + private static HikariDataSource createDataSource(String jdbcUrl) { + return createDataSource(jdbcUrl, "org.h2.Driver"); + } + + private static HikariDataSource createDataSource(String jdbcUrl, String driverClassName) { + RelationalJdbcConfiguration configuration = + TestingRelationalJdbcConfiguration.builder() + .jdbcUrl(jdbcUrl) + .driver(driverClassName) + .username("sa") + .password("") + .maximumPoolSize(1) + .build(); + return (HikariDataSource) JdbcDataSourceFactory.create(configuration).orElseThrow(); + } + + private static void writeRealmMarker(DataSource dataSource, String realmId) throws Exception { + try (Connection connection = dataSource.getConnection()) { + connection.setAutoCommit(false); + try (Statement statement = connection.createStatement()) { + statement.executeUpdate("CREATE TABLE realm_marker (realm_id VARCHAR(255))"); + } + try (PreparedStatement statement = + connection.prepareStatement("INSERT INTO realm_marker VALUES (?)")) { + statement.setString(1, realmId); + statement.executeUpdate(); + } + connection.commit(); + } + } + + private static String readRealmMarker(DataSource dataSource) throws Exception { + try (Connection connection = dataSource.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT realm_id FROM realm_marker")) { + assertThat(resultSet.next()).isTrue(); + String realmId = resultSet.getString(1); + assertThat(resultSet.next()).isFalse(); + return realmId; + } + } + + private static Path createRuntimeDriverJar(Path tempDir, String driverClassName) + throws Exception { + Path sourceRoot = tempDir.resolve("source"); + Path classesRoot = tempDir.resolve("classes"); + Path sourceFile = + sourceRoot.resolve(driverClassName.replace('.', File.separatorChar) + ".java"); + Files.createDirectories(sourceFile.getParent()); + Files.createDirectories(classesRoot); + Files.writeString(sourceFile, runtimeDriverSource(driverClassName)); + + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + assertThat(compiler).isNotNull(); + int result = + compiler.run(null, null, null, "-d", classesRoot.toString(), sourceFile.toString()); + assertThat(result).isZero(); + + Path jarFile = tempDir.resolve("runtime-loaded-jdbc-driver.jar"); + try (JarOutputStream jarOutputStream = new JarOutputStream(Files.newOutputStream(jarFile))) { + for (Path classFile : Files.walk(classesRoot).filter(Files::isRegularFile).toList()) { + JarEntry entry = + new JarEntry( + classesRoot.relativize(classFile).toString().replace(File.separatorChar, '/')); + jarOutputStream.putNextEntry(entry); + Files.copy(classFile, jarOutputStream); + jarOutputStream.closeEntry(); + } + } + return jarFile; + } + + private static String runtimeDriverSource(String driverClassName) { + int packageSeparator = driverClassName.lastIndexOf('.'); + String packageName = driverClassName.substring(0, packageSeparator); + String simpleName = driverClassName.substring(packageSeparator + 1); + return """ + package %s; + + import java.sql.Connection; + import java.sql.Driver; + import java.sql.DriverManager; + import java.sql.DriverPropertyInfo; + import java.sql.SQLException; + import java.util.Properties; + import java.util.logging.Logger; + + public final class %s implements Driver { + private static final String URL_PREFIX = "jdbc:polaris-runtime-h2:"; + + static { + try { + DriverManager.registerDriver(new %s()); + } catch (SQLException e) { + throw new ExceptionInInitializerError(e); + } + } + + @Override + public Connection connect(String url, Properties info) throws SQLException { + if (!acceptsURL(url)) { + return null; + } + try { + Class.forName("org.h2.Driver"); + } catch (ClassNotFoundException e) { + throw new SQLException(e); + } + return DriverManager.getConnection("jdbc:h2:" + url.substring(URL_PREFIX.length()), info); + } + + @Override + public boolean acceptsURL(String url) { + return url != null && url.startsWith(URL_PREFIX); + } + + @Override + public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) { + return new DriverPropertyInfo[0]; + } + + @Override + public int getMajorVersion() { + return 1; + } + + @Override + public int getMinorVersion() { + return 0; + } + + @Override + public boolean jdbcCompliant() { + return false; + } + + @Override + public Logger getParentLogger() { + return Logger.getGlobal(); + } + } + """ + .formatted(packageName, simpleName, simpleName); + } +} diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactoryTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactoryTest.java new file mode 100644 index 0000000000..eb8d179b44 --- /dev/null +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactoryTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +import jakarta.enterprise.inject.Instance; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import javax.sql.DataSource; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class JdbcMetaStoreManagerFactoryTest { + + @Mock private Instance dataSourceInstance; + + @Test + void produceDatasourceOperationsUsesQuarkusDataSourceWhenJdbcUrlIsAbsent() throws Exception { + DataSource dataSource = mockDataSource("PostgreSQL"); + when(dataSourceInstance.get()).thenReturn(dataSource); + + DatasourceOperations operations = + JdbcMetaStoreManagerFactory.produceDatasourceOperations( + dataSourceInstance, TestingRelationalJdbcConfiguration.builder().build()); + + try { + assertThat(operations.ownsDataSource()).isFalse(); + assertThat(operations.getDatabaseType()).isEqualTo(DatabaseType.POSTGRES); + verify(dataSourceInstance).get(); + } finally { + operations.close(); + } + } + + @Test + void produceDatasourceOperationsUsesPolarisDataSourceWhenJdbcUrlIsPresent() { + RelationalJdbcConfiguration configuration = + TestingRelationalJdbcConfiguration.builder() + .jdbcUrl("jdbc:h2:mem:producer_poc;DB_CLOSE_DELAY=-1") + .driver("org.h2.Driver") + .username("sa") + .password("") + .build(); + + DatasourceOperations operations = + JdbcMetaStoreManagerFactory.produceDatasourceOperations(dataSourceInstance, configuration); + + try { + assertThat(operations.ownsDataSource()).isTrue(); + assertThat(operations.getDatabaseType()).isEqualTo(DatabaseType.H2); + verify(dataSourceInstance, never()).get(); + } finally { + operations.close(); + } + } + + @Test + void closeDatasourceOperationsClosesOnlyOwnedDataSources() throws Exception { + DataSource ownedDataSource = closeableMockDataSource("PostgreSQL"); + DatasourceOperations ownedOperations = + new DatasourceOperations( + ownedDataSource, TestingRelationalJdbcConfiguration.builder().build(), true); + + ownedOperations.close(); + + verify((AutoCloseable) ownedDataSource).close(); + + DataSource externalDataSource = closeableMockDataSource("PostgreSQL"); + DatasourceOperations externalOperations = + new DatasourceOperations( + externalDataSource, TestingRelationalJdbcConfiguration.builder().build(), false); + + externalOperations.close(); + + verify((AutoCloseable) externalDataSource, never()).close(); + } + + private static DataSource mockDataSource(String databaseProductName) throws Exception { + DataSource dataSource = mock(DataSource.class); + stubConnection(dataSource, databaseProductName); + return dataSource; + } + + private static DataSource closeableMockDataSource(String databaseProductName) throws Exception { + DataSource dataSource = + mock(DataSource.class, withSettings().extraInterfaces(AutoCloseable.class)); + stubConnection(dataSource, databaseProductName); + return dataSource; + } + + private static void stubConnection(DataSource dataSource, String databaseProductName) + throws Exception { + Connection connection = mock(Connection.class); + DatabaseMetaData databaseMetaData = mock(DatabaseMetaData.class); + when(dataSource.getConnection()).thenReturn(connection); + when(connection.getMetaData()).thenReturn(databaseMetaData); + when(databaseMetaData.getDatabaseProductName()).thenReturn(databaseProductName); + } +} diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecksTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecksTest.java index 3e2656b7f3..1e27d6f8fe 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecksTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecksTest.java @@ -77,6 +77,26 @@ void jdbcWithH2ReturnsWarning() { }); } + @Test + void jdbcWithPolarisManagedH2ReturnsWarningForPolarisProperty() { + JdbcMetaStoreManagerFactory metaStoreManagerFactory = mock(JdbcMetaStoreManagerFactory.class); + DatasourceOperations operations = mock(DatasourceOperations.class); + when(datasourceOperations.get()).thenReturn(operations); + when(operations.getDatabaseType()).thenReturn(DatabaseType.H2); + when(operations.ownsDataSource()).thenReturn(true); + + ProductionReadinessCheck result = + checks.checkRelationalJdbc(metaStoreManagerFactory, datasourceOperations); + + assertThat(result.ready()).isFalse(); + assertThat(result.getErrors()) + .singleElement() + .satisfies( + error -> + assertThat(error.offendingProperty()) + .isEqualTo("polaris.persistence.relational.jdbc.jdbc-url")); + } + @Test void jdbcWithPostgresReturnsOk() { JdbcMetaStoreManagerFactory metaStoreManagerFactory = mock(JdbcMetaStoreManagerFactory.class); diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/TestingRelationalJdbcConfiguration.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/TestingRelationalJdbcConfiguration.java new file mode 100644 index 0000000000..b2fe4fd9f2 --- /dev/null +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/TestingRelationalJdbcConfiguration.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc; + +import java.util.Optional; + +final class TestingRelationalJdbcConfiguration implements RelationalJdbcConfiguration { + + private final String databaseType; + private final String jdbcUrl; + private final String driver; + private final String username; + private final String password; + private final Integer maximumPoolSize; + private final Integer minimumIdle; + private final Long connectionTimeoutInMs; + + private TestingRelationalJdbcConfiguration(Builder builder) { + this.databaseType = builder.databaseType; + this.jdbcUrl = builder.jdbcUrl; + this.driver = builder.driver; + this.username = builder.username; + this.password = builder.password; + this.maximumPoolSize = builder.maximumPoolSize; + this.minimumIdle = builder.minimumIdle; + this.connectionTimeoutInMs = builder.connectionTimeoutInMs; + } + + static Builder builder() { + return new Builder(); + } + + @Override + public Optional maxRetries() { + return Optional.of(1); + } + + @Override + public Optional maxDurationInMs() { + return Optional.of(100L); + } + + @Override + public Optional initialDelayInMs() { + return Optional.of(10L); + } + + @Override + public Optional databaseType() { + return Optional.ofNullable(databaseType); + } + + @Override + public Optional jdbcUrl() { + return Optional.ofNullable(jdbcUrl); + } + + @Override + public Optional driver() { + return Optional.ofNullable(driver); + } + + @Override + public Optional username() { + return Optional.ofNullable(username); + } + + @Override + public Optional password() { + return Optional.ofNullable(password); + } + + @Override + public Optional maximumPoolSize() { + return Optional.ofNullable(maximumPoolSize); + } + + @Override + public Optional minimumIdle() { + return Optional.ofNullable(minimumIdle); + } + + @Override + public Optional connectionTimeoutInMs() { + return Optional.ofNullable(connectionTimeoutInMs); + } + + static final class Builder { + private String databaseType; + private String jdbcUrl; + private String driver; + private String username; + private String password; + private Integer maximumPoolSize; + private Integer minimumIdle; + private Long connectionTimeoutInMs; + + private Builder() {} + + Builder databaseType(String databaseType) { + this.databaseType = databaseType; + return this; + } + + Builder jdbcUrl(String jdbcUrl) { + this.jdbcUrl = jdbcUrl; + return this; + } + + Builder driver(String driver) { + this.driver = driver; + return this; + } + + Builder username(String username) { + this.username = username; + return this; + } + + Builder password(String password) { + this.password = password; + return this; + } + + Builder maximumPoolSize(int maximumPoolSize) { + this.maximumPoolSize = maximumPoolSize; + return this; + } + + Builder minimumIdle(int minimumIdle) { + this.minimumIdle = minimumIdle; + return this; + } + + Builder connectionTimeoutInMs(long connectionTimeoutInMs) { + this.connectionTimeoutInMs = connectionTimeoutInMs; + return this; + } + + TestingRelationalJdbcConfiguration build() { + return new TestingRelationalJdbcConfiguration(this); + } + } +} diff --git a/runtime/admin/distribution/LICENSE b/runtime/admin/distribution/LICENSE index 519d8fb76f..70ab5fdc60 100644 --- a/runtime/admin/distribution/LICENSE +++ b/runtime/admin/distribution/LICENSE @@ -218,6 +218,16 @@ License: Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- +This product bundles HikariCP. + +* Maven group:artifact IDs: com.zaxxer:HikariCP + +Copyright: Copyright (C) 2013, 2014 Brett Wooldridge +Home page: https://github.com/brettwooldridge/HikariCP +License: Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt + +-------------------------------------------------------------------------------- + This product bundles and includes code from Netty. * Maven group:artifact IDs: io.netty:netty-buffer diff --git a/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java b/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java index 7eba6eaad7..476f5772ad 100644 --- a/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java +++ b/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java @@ -19,7 +19,30 @@ package org.apache.polaris.quarkus.common.config.jdbc; import io.smallrye.config.ConfigMapping; +import java.util.Optional; import org.apache.polaris.persistence.relational.jdbc.RelationalJdbcConfiguration; @ConfigMapping(prefix = "polaris.persistence.relational.jdbc") -public interface QuarkusRelationalJdbcConfiguration extends RelationalJdbcConfiguration {} +public interface QuarkusRelationalJdbcConfiguration extends RelationalJdbcConfiguration { + + @Override + Optional jdbcUrl(); + + @Override + Optional driver(); + + @Override + Optional username(); + + @Override + Optional password(); + + @Override + Optional maximumPoolSize(); + + @Override + Optional minimumIdle(); + + @Override + Optional connectionTimeoutInMs(); +} diff --git a/runtime/defaults/src/main/resources/application.properties b/runtime/defaults/src/main/resources/application.properties index d7b9196c38..d80ed5f636 100644 --- a/runtime/defaults/src/main/resources/application.properties +++ b/runtime/defaults/src/main/resources/application.properties @@ -142,6 +142,16 @@ polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"] # - nosql (beta) - NoSQL persistence backend, define the backend type via 'polaris.persistence.nosql.backend' # - relational-jdbc polaris.persistence.type=in-memory +# Optional Polaris-managed JDBC datasource for relational-jdbc persistence. When jdbc-url is set, +# Polaris creates the JDBC pool directly instead of using the Quarkus datasource extension. +# polaris.persistence.relational.jdbc.jdbc-url=jdbc:postgresql://localhost:5432/polaris +# polaris.persistence.relational.jdbc.driver=org.postgresql.Driver +# polaris.persistence.relational.jdbc.username=polaris +# polaris.persistence.relational.jdbc.password=polaris +# polaris.persistence.relational.jdbc.maximum-pool-size=10 +# For CockroachDB, set database-type=cockroachdb and use a PostgreSQL JDBC URL. +# polaris.persistence.relational.jdbc.database-type=cockroachdb +# polaris.persistence.relational.jdbc.jdbc-url=jdbc:postgresql://localhost:26257/polaris?sslmode=disable # Database backend for 'nosql' persistence-type # Available backends: # - InMemory - for testing purposes diff --git a/runtime/server/distribution/LICENSE b/runtime/server/distribution/LICENSE index 758839d15d..09c1b60e99 100644 --- a/runtime/server/distribution/LICENSE +++ b/runtime/server/distribution/LICENSE @@ -218,6 +218,16 @@ License: Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- +This product bundles HikariCP. + +* Maven group:artifact IDs: com.zaxxer:HikariCP + +Copyright: Copyright (C) 2013, 2014 Brett Wooldridge +Home page: https://github.com/brettwooldridge/HikariCP +License: Apache License 2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt + +-------------------------------------------------------------------------------- + This product bundles and includes code from Netty. * Maven group:artifact IDs: io.netty:netty-buffer diff --git a/runtime/service/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/PolarisManagedCockroachJdbcIT.java b/runtime/service/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/PolarisManagedCockroachJdbcIT.java new file mode 100644 index 0000000000..c2a450da79 --- /dev/null +++ b/runtime/service/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/PolarisManagedCockroachJdbcIT.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.test.commons.CockroachRelationalJdbcLifeCycleManagement; +import org.junit.jupiter.api.Test; + +@QuarkusTest +@TestProfile(PolarisManagedCockroachJdbcIT.Profile.class) +class PolarisManagedCockroachJdbcIT { + + @Inject DatasourceOperations datasourceOperations; + @Inject MetaStoreManagerFactory metaStoreManagerFactory; + + @Test + void managedCockroachDatasourceBootstrapsSchema() throws Exception { + assertThat(metaStoreManagerFactory).isInstanceOf(JdbcMetaStoreManagerFactory.class); + assertThat(datasourceOperations.ownsDataSource()).isTrue(); + assertThat(datasourceOperations.getDatabaseType()).isEqualTo(DatabaseType.COCKROACHDB); + + AtomicInteger schemaVersion = new AtomicInteger(); + AtomicInteger bootstrappedPrincipals = new AtomicInteger(); + + datasourceOperations.runWithinTransaction( + connection -> { + try (PreparedStatement statement = + connection.prepareStatement( + "SELECT version_value FROM version WHERE version_key = 'version'"); + ResultSet resultSet = statement.executeQuery()) { + assertThat(resultSet.next()).isTrue(); + schemaVersion.set(resultSet.getInt(1)); + } + + try (PreparedStatement statement = + connection.prepareStatement( + "SELECT COUNT(*) FROM principal_authentication_data"); + ResultSet resultSet = statement.executeQuery()) { + assertThat(resultSet.next()).isTrue(); + bootstrappedPrincipals.set(resultSet.getInt(1)); + } + + return true; + }); + + assertThat(schemaVersion.get()).isEqualTo(4); + assertThat(bootstrappedPrincipals.get()).isPositive(); + } + + public static class Profile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + Map config = new HashMap<>(); + config.put("polaris.persistence.auto-bootstrap-types", "relational-jdbc"); + config.put("polaris.persistence.relational.jdbc.database-type", "cockroachdb"); + return Map.copyOf(config); + } + + @Override + public List testResources() { + return List.of( + new QuarkusTestProfile.TestResourceEntry( + CockroachRelationalJdbcLifeCycleManagement.class, + Map.of( + CockroachRelationalJdbcLifeCycleManagement.POLARIS_MANAGED_DATASOURCE, "true"))); + } + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/persistence/relational/jdbc/PolarisManagedJdbcIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/persistence/relational/jdbc/PolarisManagedJdbcIntegrationTest.java new file mode 100644 index 0000000000..c77f94672a --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/persistence/relational/jdbc/PolarisManagedJdbcIntegrationTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.common.collect.ImmutableMap; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.TestProfile; +import jakarta.inject.Inject; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.service.Profiles; +import org.junit.jupiter.api.Test; + +@QuarkusTest +@TestProfile(PolarisManagedJdbcIntegrationTest.Profile.class) +class PolarisManagedJdbcIntegrationTest { + + @Inject DatasourceOperations datasourceOperations; + @Inject MetaStoreManagerFactory metaStoreManagerFactory; + + @Test + void managedJdbcDatasourceBootstrapsSchema() throws Exception { + assertThat(metaStoreManagerFactory).isInstanceOf(JdbcMetaStoreManagerFactory.class); + assertThat(datasourceOperations.ownsDataSource()).isTrue(); + assertThat(datasourceOperations.getDatabaseType()).isEqualTo(DatabaseType.H2); + + AtomicInteger schemaVersion = new AtomicInteger(); + AtomicInteger bootstrappedPrincipals = new AtomicInteger(); + + datasourceOperations.runWithinTransaction( + connection -> { + try (PreparedStatement statement = + connection.prepareStatement( + "SELECT version_value FROM version WHERE version_key = 'version'"); + ResultSet resultSet = statement.executeQuery()) { + assertThat(resultSet.next()).isTrue(); + schemaVersion.set(resultSet.getInt(1)); + } + + try (PreparedStatement statement = + connection.prepareStatement( + "SELECT COUNT(*) FROM principal_authentication_data"); + ResultSet resultSet = statement.executeQuery()) { + assertThat(resultSet.next()).isTrue(); + bootstrappedPrincipals.set(resultSet.getInt(1)); + } + + return true; + }); + + assertThat(schemaVersion.get()).isEqualTo(4); + assertThat(bootstrappedPrincipals.get()).isPositive(); + } + + public static class Profile extends Profiles.DefaultProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .put("quarkus.datasource.active", "false") + .put("polaris.persistence.type", "relational-jdbc") + .put("polaris.persistence.auto-bootstrap-types", "relational-jdbc") + .put( + "polaris.persistence.relational.jdbc.jdbc-url", + "jdbc:h2:mem:polaris_managed_jdbc_it;DB_CLOSE_DELAY=-1;" + + "MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE") + .put("polaris.persistence.relational.jdbc.driver", "org.h2.Driver") + .put("polaris.persistence.relational.jdbc.username", "sa") + .put("polaris.persistence.relational.jdbc.maximum-pool-size", "2") + .put("polaris.persistence.relational.jdbc.minimum-idle", "1") + .put("polaris.persistence.relational.jdbc.connection-timeout-in-ms", "1000") + .build(); + } + } +} diff --git a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/CockroachRelationalJdbcLifeCycleManagement.java b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/CockroachRelationalJdbcLifeCycleManagement.java index 6a1577c7ca..f0165fc465 100644 --- a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/CockroachRelationalJdbcLifeCycleManagement.java +++ b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/CockroachRelationalJdbcLifeCycleManagement.java @@ -22,20 +22,24 @@ import io.quarkus.test.common.DevServicesContext; import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import java.util.HashMap; import java.util.Map; import org.testcontainers.cockroachdb.CockroachContainer; public class CockroachRelationalJdbcLifeCycleManagement implements QuarkusTestResourceLifecycleManager, DevServicesContext.ContextAware { public static final String INIT_SCRIPT = "init-script"; + public static final String POLARIS_MANAGED_DATASOURCE = "polaris-managed-datasource"; private CockroachContainer cockroach; private String initScript; + private boolean polarisManagedDatasource; private DevServicesContext context; @Override public void init(Map initArgs) { initScript = initArgs.get(INIT_SCRIPT); + polarisManagedDatasource = Boolean.parseBoolean(initArgs.get(POLARIS_MANAGED_DATASOURCE)); } @Override @@ -56,23 +60,27 @@ public Map start() { // CockroachDB uses PostgreSQL JDBC driver and wire protocol // Explicitly configure database type as cockroachdb for proper identification - return Map.of( - "polaris.persistence.type", - "relational-jdbc", - "polaris.persistence.relational.jdbc.database-type", - "cockroachdb", - "polaris.persistence.relational.jdbc.max-retries", - "2", - "quarkus.datasource.db-kind", - "postgresql", - "quarkus.datasource.jdbc.url", - cockroach.getJdbcUrl(), - "quarkus.datasource.username", - cockroach.getUsername(), - "quarkus.datasource.password", - cockroach.getPassword(), - "quarkus.datasource.jdbc.initial-size", - "10"); + Map config = new HashMap<>(); + config.put("polaris.persistence.type", "relational-jdbc"); + config.put("polaris.persistence.relational.jdbc.database-type", "cockroachdb"); + config.put("polaris.persistence.relational.jdbc.max-retries", "2"); + + if (polarisManagedDatasource) { + config.put("quarkus.datasource.active", "false"); + config.put("polaris.persistence.relational.jdbc.jdbc-url", cockroach.getJdbcUrl()); + config.put("polaris.persistence.relational.jdbc.driver", "org.postgresql.Driver"); + config.put("polaris.persistence.relational.jdbc.username", cockroach.getUsername()); + config.put("polaris.persistence.relational.jdbc.password", cockroach.getPassword()); + config.put("polaris.persistence.relational.jdbc.maximum-pool-size", "10"); + } else { + config.put("quarkus.datasource.db-kind", "postgresql"); + config.put("quarkus.datasource.jdbc.url", cockroach.getJdbcUrl()); + config.put("quarkus.datasource.username", cockroach.getUsername()); + config.put("quarkus.datasource.password", cockroach.getPassword()); + config.put("quarkus.datasource.jdbc.initial-size", "10"); + } + + return Map.copyOf(config); } @Override diff --git a/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md b/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md index e50f686542..97785dd7fa 100644 --- a/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md +++ b/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md @@ -29,3 +29,10 @@ build: | `polaris.persistence.relational.jdbc.max-duration-in-ms` | | `long` | | | `polaris.persistence.relational.jdbc.initial-delay-in-ms` | | `long` | | | `polaris.persistence.relational.jdbc.database-type` | | `string` | | +| `polaris.persistence.relational.jdbc.jdbc-url` | | `string` | Optional JDBC URL for a Polaris-managed datasource. When set, Polaris creates the JDBC pool directly instead of using the Quarkus datasource extension. | +| `polaris.persistence.relational.jdbc.driver` | | `string` | Optional JDBC driver class name for the Polaris-managed datasource, for example `org.postgresql.Driver`. | +| `polaris.persistence.relational.jdbc.username` | | `string` | Optional username for the Polaris-managed datasource. | +| `polaris.persistence.relational.jdbc.password` | | `string` | Optional password for the Polaris-managed datasource. | +| `polaris.persistence.relational.jdbc.maximum-pool-size` | | `int` | Optional maximum connection pool size for the Polaris-managed datasource. | +| `polaris.persistence.relational.jdbc.minimum-idle` | | `int` | Optional minimum idle connection count for the Polaris-managed datasource. | +| `polaris.persistence.relational.jdbc.connection-timeout-in-ms` | | `long` | Optional connection timeout in milliseconds for the Polaris-managed datasource. | diff --git a/site/content/in-dev/unreleased/configuration/configuring-polaris-for-production/_index.md b/site/content/in-dev/unreleased/configuration/configuring-polaris-for-production/_index.md index 085248ff45..e78a2b6d9c 100644 --- a/site/content/in-dev/unreleased/configuration/configuring-polaris-for-production/_index.md +++ b/site/content/in-dev/unreleased/configuration/configuring-polaris-for-production/_index.md @@ -140,8 +140,9 @@ when the server is restarted; it is also unusable when multiple Polaris replicas {{< /alert >}} To enable a durable metastore, configure your system to use the Relational JDBC-backed metastore. -This implementation leverages Quarkus for datasource management and supports configuration through -environment variables or JVM -D flags at startup. For more information, refer to the [Quarkus configuration reference](https://quarkus.io/guides/config-reference#env-file). +This implementation can use Quarkus datasource management or a Polaris-managed JDBC connection pool +and supports configuration through environment variables or JVM -D flags at startup. For more +information, refer to the [Quarkus configuration reference](https://quarkus.io/guides/config-reference#env-file). Configure the metastore by setting the following ENV variables: @@ -154,9 +155,9 @@ QUARKUS_DATASOURCE_JDBC_URL= ``` -The relational JDBC metastore is a Quarkus-managed datasource and only supports Postgres and H2 as of now. -Please refer to the documentation here: -[Configure data sources in Quarkus](https://quarkus.io/guides/datasource) +The relational JDBC metastore supports PostgreSQL and CockroachDB for production use. H2 is available +for testing and development only. For Quarkus-managed datasource configuration, refer to +[Configure data sources in Quarkus](https://quarkus.io/guides/datasource). {{< alert important >}} Be sure to secure your metastore backend since it will be storing sensitive data and catalog diff --git a/site/content/in-dev/unreleased/metastores/relational-jdbc.md b/site/content/in-dev/unreleased/metastores/relational-jdbc.md index 037cb0fafe..0363fb078c 100644 --- a/site/content/in-dev/unreleased/metastores/relational-jdbc.md +++ b/site/content/in-dev/unreleased/metastores/relational-jdbc.md @@ -22,12 +22,18 @@ type: docs weight: 100 --- -This implementation leverages Quarkus for datasource management and supports configuration through -environment variables or JVM -D flags at startup. For more information, refer to the [Quarkus configuration reference](https://quarkus.io/guides/config-reference#env-file). +The Relational JDBC metastore supports PostgreSQL, CockroachDB, and H2. H2 is intended for testing +and development only. -We have 2 options for configuring the persistence backend: +Polaris can use either a Quarkus-managed datasource or a Polaris-managed JDBC connection pool. When +`polaris.persistence.relational.jdbc.jdbc-url` is set, Polaris creates the JDBC pool directly; +otherwise, Polaris uses the Quarkus datasource configuration. Both forms support configuration +through environment variables or JVM `-D` flags at startup. For more information, refer to the +[Quarkus configuration reference](https://quarkus.io/guides/config-reference#env-file). -## 1. Relational JDBC metastore with username and password +We have 3 options for configuring the persistence backend: + +## 1. Quarkus-managed Relational JDBC metastore with username and password Using environment variables: @@ -43,12 +49,48 @@ Using properties file: ```properties polaris.persistence.type=relational-jdbc -quarkus.datasource.jdbc.username= -quarkus.datasource.jdbc.password= -quarkus.datasource.jdbc.jdbc-url= +quarkus.datasource.username= +quarkus.datasource.password= +quarkus.datasource.jdbc.url= +``` + +## 2. Polaris-managed Relational JDBC metastore with username and password + +Using environment variables: + +```properties +POLARIS_PERSISTENCE_TYPE=relational-jdbc + +POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATABASE_TYPE=postgresql +POLARIS_PERSISTENCE_RELATIONAL_JDBC_JDBC_URL= +POLARIS_PERSISTENCE_RELATIONAL_JDBC_DRIVER=org.postgresql.Driver +POLARIS_PERSISTENCE_RELATIONAL_JDBC_USERNAME= +POLARIS_PERSISTENCE_RELATIONAL_JDBC_PASSWORD= +``` + +Using properties file: + +```properties +polaris.persistence.type=relational-jdbc +polaris.persistence.relational.jdbc.database-type=postgresql +polaris.persistence.relational.jdbc.jdbc-url= +polaris.persistence.relational.jdbc.driver=org.postgresql.Driver +polaris.persistence.relational.jdbc.username= +polaris.persistence.relational.jdbc.password= ``` -## 2. AWS Aurora PostgreSQL metastore using IAM AWS authentication +For CockroachDB, use the PostgreSQL JDBC driver and set the database type explicitly: + +```properties +polaris.persistence.type=relational-jdbc +polaris.persistence.relational.jdbc.database-type=cockroachdb +polaris.persistence.relational.jdbc.jdbc-url=jdbc:postgresql://cockroachdb:26257/polaris?sslmode=disable +polaris.persistence.relational.jdbc.driver=org.postgresql.Driver +polaris.persistence.relational.jdbc.username= +polaris.persistence.relational.jdbc.password= +``` + +## 3. AWS Aurora PostgreSQL metastore using IAM AWS authentication ```properties polaris.persistence.type=relational-jdbc @@ -68,11 +110,9 @@ quarkus.rds.credentials-provider.aws.port=6160 This is the basic configuration. For more details, please refer to the [Quarkus plugin documentation](https://docs.quarkiverse.io/quarkus-amazon-services/dev/amazon-rds.html#_configuration_reference). -The Relational JDBC metastore currently relies on a Quarkus-managed datasource and supports only PostgresSQL and H2 databases. At this time, official documentation is provided exclusively for usage with PostgreSQL. -Please refer to the documentation here: -[Configure data sources in Quarkus](https://quarkus.io/guides/datasource). - -Additionally, the retries can be configured via `polaris.persistence.relational.jdbc.*` properties; please refer to the [Configuring Polaris]({{% ref "../configuration" %}}) section. +The retries and Polaris-managed datasource pool can be configured via +`polaris.persistence.relational.jdbc.*` properties; please refer to the +[Configuring Polaris]({{% ref "../configuration" %}}) section. ## Bootstrapping Polaris @@ -100,4 +140,8 @@ java \ -jar polaris-admin-tool.jar bootstrap -r -c ,, ``` +When using the Polaris-managed JDBC pool, pass the corresponding +`polaris.persistence.relational.jdbc.*` settings to the admin tool instead of the Quarkus datasource +settings. + For more details on the bootstrap command and other administrative operations, see the [Admin Tool]({{% ref "../admin-tool" %}}) documentation. From 4ece8d954099b2bcc37db6e63bb2371336a2d02b Mon Sep 17 00:00:00 2001 From: Yufei Date: Mon, 6 Jul 2026 18:19:37 -0700 Subject: [PATCH 2/3] Add JDBC driver drop-in loading for managed datasources Support loading third-party JDBC driver jars from a configured driver directory when Polaris creates the relational JDBC datasource directly. For binary distributions, include visible server and admin jdbc-drivers directories and document the drop-in workflow. Add tests for explicit driver-directory loading and the default quarkus-run.jar sibling directory layout. --- .../jdbc/JdbcDataSourceFactory.java | 167 +++++++++++++++++- .../jdbc/RelationalJdbcConfiguration.java | 5 + .../jdbc/JdbcDataSourceFactoryTest.java | 80 +++++++-- .../TestingRelationalJdbcConfiguration.java | 13 ++ .../QuarkusRelationalJdbcConfiguration.java | 3 + .../src/main/resources/application.properties | 3 + runtime/distribution/README.md | 4 +- runtime/distribution/build.gradle.kts | 2 + runtime/distribution/jdbc-drivers/README.md | 22 +++ ...rye-polaris_persistence_relational_jdbc.md | 1 + .../unreleased/metastores/relational-jdbc.md | 8 + 11 files changed, 288 insertions(+), 20 deletions(-) create mode 100644 runtime/distribution/jdbc-drivers/README.md diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactory.java index a5c0c4bc14..4299c0a7d5 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactory.java @@ -20,11 +20,25 @@ import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; import java.util.Optional; +import java.util.stream.Stream; import javax.sql.DataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; final class JdbcDataSourceFactory { + private static final Logger LOGGER = LoggerFactory.getLogger(JdbcDataSourceFactory.class); + + private static final String DEFAULT_DRIVER_DIRECTORY = "jdbc-drivers"; private static final String POOL_NAME = "polaris-relational-jdbc"; private JdbcDataSourceFactory() {} @@ -35,9 +49,39 @@ static Optional create(RelationalJdbcConfiguration configuration) { return Optional.empty(); } + Optional driverClassLoader = createDriverClassLoader(configuration); + if (driverClassLoader.isPresent()) { + return Optional.of( + createHikariDataSource(configuration, jdbcUrl.get(), driverClassLoader.get())); + } + + return Optional.of(new HikariDataSource(createHikariConfig(configuration, jdbcUrl.get()))); + } + + private static HikariDataSource createHikariDataSource( + RelationalJdbcConfiguration configuration, String jdbcUrl, URLClassLoader driverClassLoader) { + ClassLoader previousContextClassLoader = Thread.currentThread().getContextClassLoader(); + boolean success = false; + try { + Thread.currentThread().setContextClassLoader(driverClassLoader); + HikariDataSource dataSource = + new DriverClassLoaderHikariDataSource( + createHikariConfig(configuration, jdbcUrl), driverClassLoader); + success = true; + return dataSource; + } finally { + Thread.currentThread().setContextClassLoader(previousContextClassLoader); + if (!success) { + closeDriverClassLoader(driverClassLoader); + } + } + } + + private static HikariConfig createHikariConfig( + RelationalJdbcConfiguration configuration, String jdbcUrl) { HikariConfig hikariConfig = new HikariConfig(); hikariConfig.setPoolName(POOL_NAME); - hikariConfig.setJdbcUrl(jdbcUrl.get()); + hikariConfig.setJdbcUrl(jdbcUrl); nonBlank(configuration.driver()).ifPresent(hikariConfig::setDriverClassName); nonBlank(configuration.username()).ifPresent(hikariConfig::setUsername); configuration.password().ifPresent(hikariConfig::setPassword); @@ -45,10 +89,129 @@ static Optional create(RelationalJdbcConfiguration configuration) { configuration.minimumIdle().ifPresent(hikariConfig::setMinimumIdle); configuration.connectionTimeoutInMs().ifPresent(hikariConfig::setConnectionTimeout); - return Optional.of(new HikariDataSource(hikariConfig)); + return hikariConfig; + } + + private static Optional createDriverClassLoader( + RelationalJdbcConfiguration configuration) { + Optional configuredDriverDirectory = nonBlank(configuration.driverDirectory()); + Optional driverDirectory = + configuredDriverDirectory + .map(Path::of) + .or(() -> defaultDriverDirectory().filter(Files::isDirectory)); + + if (driverDirectory.isEmpty()) { + return Optional.empty(); + } + + Path directory = driverDirectory.get(); + if (!Files.isDirectory(directory)) { + throw new IllegalArgumentException( + "Configured JDBC driver directory does not exist: " + directory); + } + + List driverJars = driverJars(directory); + if (driverJars.isEmpty()) { + if (configuredDriverDirectory.isPresent()) { + throw new IllegalArgumentException( + "Configured JDBC driver directory does not contain any .jar files: " + directory); + } + return Optional.empty(); + } + + LOGGER.info("Loading JDBC driver jars from {}", directory); + URL[] urls = driverJars.stream().map(JdbcDataSourceFactory::toUrl).toArray(URL[]::new); + ClassLoader parent = Thread.currentThread().getContextClassLoader(); + if (parent == null) { + parent = JdbcDataSourceFactory.class.getClassLoader(); + } + return Optional.of(new URLClassLoader(urls, parent)); + } + + private static Optional defaultDriverDirectory() { + String classPath = System.getProperty("java.class.path", ""); + for (String classPathEntry : classPath.split(System.getProperty("path.separator"))) { + if (classPathEntry.isEmpty()) { + continue; + } + Path path = Path.of(classPathEntry); + Path fileName = path.getFileName(); + if (fileName != null && "quarkus-run.jar".equals(fileName.toString())) { + Path parent = path.getParent(); + return Optional.of( + parent == null + ? Path.of(DEFAULT_DRIVER_DIRECTORY) + : parent.resolve(DEFAULT_DRIVER_DIRECTORY)); + } + } + return Optional.empty(); + } + + private static List driverJars(Path driverDirectory) { + try (Stream entries = Files.list(driverDirectory)) { + return entries + .filter(Files::isRegularFile) + .filter(path -> path.getFileName().toString().toLowerCase(Locale.ROOT).endsWith(".jar")) + .sorted(Comparator.comparing(path -> path.getFileName().toString())) + .toList(); + } catch (IOException e) { + throw new RuntimeException("Failed to list JDBC driver directory: " + driverDirectory, e); + } + } + + private static URL toUrl(Path driverJar) { + try { + return driverJar.toUri().toURL(); + } catch (IOException e) { + throw new RuntimeException("Failed to load JDBC driver jar: " + driverJar, e); + } + } + + private static void closeDriverClassLoader(URLClassLoader driverClassLoader) { + try { + driverClassLoader.close(); + } catch (IOException e) { + throw new RuntimeException("Failed to close JDBC driver classloader", e); + } } private static Optional nonBlank(Optional value) { return value.map(String::trim).filter(s -> !s.isEmpty()); } + + private static final class DriverClassLoaderHikariDataSource extends HikariDataSource { + private final URLClassLoader driverClassLoader; + + private DriverClassLoaderHikariDataSource( + HikariConfig configuration, URLClassLoader driverClassLoader) { + super(configuration); + this.driverClassLoader = driverClassLoader; + } + + @Override + public void close() { + RuntimeException closeFailure = null; + try { + super.close(); + } catch (RuntimeException e) { + closeFailure = e; + } + + try { + driverClassLoader.close(); + } catch (IOException e) { + RuntimeException classLoaderFailure = + new RuntimeException("Failed to close JDBC driver classloader", e); + if (closeFailure != null) { + closeFailure.addSuppressed(classLoaderFailure); + } else { + closeFailure = classLoaderFailure; + } + } + + if (closeFailure != null) { + throw closeFailure; + } + } + } } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java index 652f77f7b9..30f7aa4e08 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java @@ -46,6 +46,11 @@ default Optional driver() { return Optional.empty(); } + /** Directory containing JDBC driver jars for the Polaris-managed datasource. */ + default Optional driverDirectory() { + return Optional.empty(); + } + /** Database username used when Polaris creates and owns the datasource. */ default Optional username() { return Optional.empty(); diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactoryTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactoryTest.java index 8808faa791..02c730f16b 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactoryTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcDataSourceFactoryTest.java @@ -23,8 +23,6 @@ import com.zaxxer.hikari.HikariDataSource; import java.io.File; -import java.net.URL; -import java.net.URLClassLoader; import java.nio.file.Files; import java.nio.file.Path; import java.sql.Connection; @@ -102,7 +100,9 @@ void createBuildsIndependentDataSourcesForDifferentRealmConfigurations() throws @Test void createUsesJdbcDriverLoadedFromRuntimeJar(@TempDir Path tempDir) throws Exception { String driverClassName = "runtime.jdbc.RuntimeLoadedJdbcDriver"; - Path driverJar = createRuntimeDriverJar(tempDir, driverClassName); + Path driverDirectory = tempDir.resolve("jdbc-drivers"); + Files.createDirectories(driverDirectory); + createRuntimeDriverJar(driverDirectory, driverClassName); assertThatThrownBy( () -> @@ -110,40 +110,86 @@ void createUsesJdbcDriverLoadedFromRuntimeJar(@TempDir Path tempDir) throws Exce driverClassName, false, JdbcDataSourceFactoryTest.class.getClassLoader())) .isInstanceOf(ClassNotFoundException.class); - ClassLoader previousContextClassLoader = Thread.currentThread().getContextClassLoader(); - try (URLClassLoader runtimeClassLoader = - new URLClassLoader(new URL[] {driverJar.toUri().toURL()}, previousContextClassLoader)) { - Thread.currentThread().setContextClassLoader(runtimeClassLoader); + HikariDataSource dataSource = + createDataSource( + "jdbc:polaris-runtime-h2:mem:runtime_loaded_driver_poc;DB_CLOSE_DELAY=-1", + driverClassName, + driverDirectory); + + try { + writeRealmMarker(dataSource, "runtime-loaded-driver"); + + assertThat(readRealmMarker(dataSource)).isEqualTo("runtime-loaded-driver"); + } finally { + dataSource.close(); + } + } + + @Test + void createUsesJdbcDriverLoadedFromDefaultRuntimeDirectory(@TempDir Path tempDir) + throws Exception { + String driverClassName = "runtime.jdbc.DefaultRuntimeLoadedJdbcDriver"; + Path serverDirectory = tempDir.resolve("server"); + Path launcherJar = serverDirectory.resolve("quarkus-run.jar"); + Path driverDirectory = serverDirectory.resolve("jdbc-drivers"); + Files.createDirectories(driverDirectory); + Files.createFile(launcherJar); + createRuntimeDriverJar(driverDirectory, driverClassName); + + String previousClassPath = System.getProperty("java.class.path"); + try { + System.setProperty("java.class.path", launcherJar.toString()); HikariDataSource dataSource = createDataSource( - "jdbc:polaris-runtime-h2:mem:runtime_loaded_driver_poc;DB_CLOSE_DELAY=-1", - driverClassName); + "jdbc:polaris-runtime-h2:mem:default_runtime_loaded_driver_poc;DB_CLOSE_DELAY=-1", + driverClassName, + null); try { - writeRealmMarker(dataSource, "runtime-loaded-driver"); + writeRealmMarker(dataSource, "default-runtime-loaded-driver"); - assertThat(readRealmMarker(dataSource)).isEqualTo("runtime-loaded-driver"); + assertThat(readRealmMarker(dataSource)).isEqualTo("default-runtime-loaded-driver"); } finally { dataSource.close(); } } finally { - Thread.currentThread().setContextClassLoader(previousContextClassLoader); + System.setProperty("java.class.path", previousClassPath); } } + @Test + void createRejectsConfiguredDriverDirectoryWithoutJars(@TempDir Path tempDir) throws Exception { + Path driverDirectory = tempDir.resolve("jdbc-drivers"); + Files.createDirectories(driverDirectory); + RelationalJdbcConfiguration configuration = + TestingRelationalJdbcConfiguration.builder() + .jdbcUrl("jdbc:h2:mem:missing_runtime_driver;DB_CLOSE_DELAY=-1") + .driver("missing.Driver") + .driverDirectory(driverDirectory.toString()) + .build(); + + assertThatThrownBy(() -> JdbcDataSourceFactory.create(configuration)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("does not contain any .jar files"); + } + private static HikariDataSource createDataSource(String jdbcUrl) { - return createDataSource(jdbcUrl, "org.h2.Driver"); + return createDataSource(jdbcUrl, "org.h2.Driver", null); } - private static HikariDataSource createDataSource(String jdbcUrl, String driverClassName) { - RelationalJdbcConfiguration configuration = + private static HikariDataSource createDataSource( + String jdbcUrl, String driverClassName, Path driverDirectory) { + TestingRelationalJdbcConfiguration.Builder builder = TestingRelationalJdbcConfiguration.builder() .jdbcUrl(jdbcUrl) .driver(driverClassName) .username("sa") .password("") - .maximumPoolSize(1) - .build(); + .maximumPoolSize(1); + if (driverDirectory != null) { + builder.driverDirectory(driverDirectory.toString()); + } + RelationalJdbcConfiguration configuration = builder.build(); return (HikariDataSource) JdbcDataSourceFactory.create(configuration).orElseThrow(); } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/TestingRelationalJdbcConfiguration.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/TestingRelationalJdbcConfiguration.java index b2fe4fd9f2..5e981740a1 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/TestingRelationalJdbcConfiguration.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/TestingRelationalJdbcConfiguration.java @@ -25,6 +25,7 @@ final class TestingRelationalJdbcConfiguration implements RelationalJdbcConfigur private final String databaseType; private final String jdbcUrl; private final String driver; + private final String driverDirectory; private final String username; private final String password; private final Integer maximumPoolSize; @@ -35,6 +36,7 @@ private TestingRelationalJdbcConfiguration(Builder builder) { this.databaseType = builder.databaseType; this.jdbcUrl = builder.jdbcUrl; this.driver = builder.driver; + this.driverDirectory = builder.driverDirectory; this.username = builder.username; this.password = builder.password; this.maximumPoolSize = builder.maximumPoolSize; @@ -76,6 +78,11 @@ public Optional driver() { return Optional.ofNullable(driver); } + @Override + public Optional driverDirectory() { + return Optional.ofNullable(driverDirectory); + } + @Override public Optional username() { return Optional.ofNullable(username); @@ -105,6 +112,7 @@ static final class Builder { private String databaseType; private String jdbcUrl; private String driver; + private String driverDirectory; private String username; private String password; private Integer maximumPoolSize; @@ -128,6 +136,11 @@ Builder driver(String driver) { return this; } + Builder driverDirectory(String driverDirectory) { + this.driverDirectory = driverDirectory; + return this; + } + Builder username(String username) { this.username = username; return this; diff --git a/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java b/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java index 476f5772ad..943add15bc 100644 --- a/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java +++ b/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java @@ -31,6 +31,9 @@ public interface QuarkusRelationalJdbcConfiguration extends RelationalJdbcConfig @Override Optional driver(); + @Override + Optional driverDirectory(); + @Override Optional username(); diff --git a/runtime/defaults/src/main/resources/application.properties b/runtime/defaults/src/main/resources/application.properties index d80ed5f636..7dfb17fe40 100644 --- a/runtime/defaults/src/main/resources/application.properties +++ b/runtime/defaults/src/main/resources/application.properties @@ -146,6 +146,9 @@ polaris.persistence.type=in-memory # Polaris creates the JDBC pool directly instead of using the Quarkus datasource extension. # polaris.persistence.relational.jdbc.jdbc-url=jdbc:postgresql://localhost:5432/polaris # polaris.persistence.relational.jdbc.driver=org.postgresql.Driver +# Optional directory of JDBC driver jars. In binary distributions, put server driver jars under +# server/jdbc-drivers or set this property to another path. +# polaris.persistence.relational.jdbc.driver-directory=jdbc-drivers # polaris.persistence.relational.jdbc.username=polaris # polaris.persistence.relational.jdbc.password=polaris # polaris.persistence.relational.jdbc.maximum-pool-size=10 diff --git a/runtime/distribution/README.md b/runtime/distribution/README.md index a372124e13..68957e4dc8 100644 --- a/runtime/distribution/README.md +++ b/runtime/distribution/README.md @@ -33,10 +33,12 @@ polaris-distribution-@version@/ ├── NOTICE ├── README.md ├── admin/ # Admin tool files +│ └── jdbc-drivers/ # JDBC driver jars for the Polaris-managed datasource ├── bin/ # Executable scripts │ ├── admin │ └── server └── server/ # Server files + └── jdbc-drivers/ # JDBC driver jars for the Polaris-managed datasource ``` ## Usage @@ -76,4 +78,4 @@ bin/server ``` For more details on configuration, please refer to the Polaris documentation: -https://polaris.apache.org/ \ No newline at end of file +https://polaris.apache.org/ diff --git a/runtime/distribution/build.gradle.kts b/runtime/distribution/build.gradle.kts index a85e78845a..d67de211da 100644 --- a/runtime/distribution/build.gradle.kts +++ b/runtime/distribution/build.gradle.kts @@ -73,9 +73,11 @@ distributions { contents { // Copy admin distribution contents into("admin") { from(adminDistribution) { exclude("quarkus-app-dependencies.txt") } } + into("admin/jdbc-drivers") { from("jdbc-drivers") } // Copy server distribution contents into("server") { from(serverDistribution) { exclude("quarkus-app-dependencies.txt") } } + into("server/jdbc-drivers") { from("jdbc-drivers") } // Copy scripts to bin directory into("bin") { diff --git a/runtime/distribution/jdbc-drivers/README.md b/runtime/distribution/jdbc-drivers/README.md new file mode 100644 index 0000000000..876f6d9f85 --- /dev/null +++ b/runtime/distribution/jdbc-drivers/README.md @@ -0,0 +1,22 @@ + + +# JDBC Driver Drop-In Directory + +Place JDBC driver jars in this directory when using the Polaris-managed relational JDBC datasource. diff --git a/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md b/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md index 97785dd7fa..849b7e2a2f 100644 --- a/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md +++ b/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md @@ -31,6 +31,7 @@ build: | `polaris.persistence.relational.jdbc.database-type` | | `string` | | | `polaris.persistence.relational.jdbc.jdbc-url` | | `string` | Optional JDBC URL for a Polaris-managed datasource. When set, Polaris creates the JDBC pool directly instead of using the Quarkus datasource extension. | | `polaris.persistence.relational.jdbc.driver` | | `string` | Optional JDBC driver class name for the Polaris-managed datasource, for example `org.postgresql.Driver`. | +| `polaris.persistence.relational.jdbc.driver-directory` | | `string` | Optional directory of JDBC driver jars loaded for the Polaris-managed datasource. When unset, Polaris looks for `jdbc-drivers` beside `quarkus-run.jar` in binary distributions. | | `polaris.persistence.relational.jdbc.username` | | `string` | Optional username for the Polaris-managed datasource. | | `polaris.persistence.relational.jdbc.password` | | `string` | Optional password for the Polaris-managed datasource. | | `polaris.persistence.relational.jdbc.maximum-pool-size` | | `int` | Optional maximum connection pool size for the Polaris-managed datasource. | diff --git a/site/content/in-dev/unreleased/metastores/relational-jdbc.md b/site/content/in-dev/unreleased/metastores/relational-jdbc.md index 0363fb078c..71c5d768fd 100644 --- a/site/content/in-dev/unreleased/metastores/relational-jdbc.md +++ b/site/content/in-dev/unreleased/metastores/relational-jdbc.md @@ -79,6 +79,14 @@ polaris.persistence.relational.jdbc.username= polaris.persistence.relational.jdbc.password= ``` +When using the binary distribution, place third-party JDBC driver jars under +`server/jdbc-drivers`. Polaris loads jars from that directory before creating the managed +datasource. If the jars live elsewhere, set +`polaris.persistence.relational.jdbc.driver-directory=/path/to/jdbc-drivers` or +`POLARIS_PERSISTENCE_RELATIONAL_JDBC_DRIVER_DIRECTORY=/path/to/jdbc-drivers`. The admin tool runs +from `admin`, so bootstrap and purge commands need the same driver jar under `admin/jdbc-drivers` or +the same explicit `driver-directory` setting. + For CockroachDB, use the PostgreSQL JDBC driver and set the database type explicitly: ```properties From 56ef10f49a248121f7d6fae0dfa1765c6bde6015 Mon Sep 17 00:00:00 2001 From: Yufei Date: Mon, 6 Jul 2026 22:28:14 -0700 Subject: [PATCH 3/3] Update generated JDBC configuration docs Move Polaris-managed JDBC datasource property descriptions into the Quarkus config mapping so generated config docs stay current. Regenerate the relational JDBC config section with copyConfigSectionsToSite. --- .../jdbc/QuarkusRelationalJdbcConfiguration.java | 14 ++++++++++++++ ...allrye-polaris_persistence_relational_jdbc.md | 16 ++++++++-------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java b/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java index 943add15bc..9c2acfc7fb 100644 --- a/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java +++ b/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java @@ -25,27 +25,41 @@ @ConfigMapping(prefix = "polaris.persistence.relational.jdbc") public interface QuarkusRelationalJdbcConfiguration extends RelationalJdbcConfiguration { + /** + * Optional JDBC URL for a Polaris-managed datasource. When set, Polaris creates the JDBC pool + * directly instead of using the Quarkus datasource extension. + */ @Override Optional jdbcUrl(); + /** Optional JDBC driver class name for the Polaris-managed datasource. */ @Override Optional driver(); + /** + * Optional directory of JDBC driver jars loaded for the Polaris-managed datasource. When unset, + * Polaris looks for {@code jdbc-drivers} beside {@code quarkus-run.jar} in binary distributions. + */ @Override Optional driverDirectory(); + /** Optional username for the Polaris-managed datasource. */ @Override Optional username(); + /** Optional password for the Polaris-managed datasource. */ @Override Optional password(); + /** Optional maximum connection pool size for the Polaris-managed datasource. */ @Override Optional maximumPoolSize(); + /** Optional minimum idle connection count for the Polaris-managed datasource. */ @Override Optional minimumIdle(); + /** Optional connection timeout in milliseconds for the Polaris-managed datasource. */ @Override Optional connectionTimeoutInMs(); } diff --git a/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md b/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md index 849b7e2a2f..882386f9ab 100644 --- a/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md +++ b/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md @@ -25,15 +25,15 @@ build: | Property | Default Value | Type | Description | |----------|---------------|------|-------------| +| `polaris.persistence.relational.jdbc.jdbc-url` | | `string` | Optional JDBC URL for a Polaris-managed datasource. When set, Polaris creates the JDBC pool directly instead of using the Quarkus datasource extension. | +| `polaris.persistence.relational.jdbc.driver` | | `string` | Optional JDBC driver class name for the Polaris-managed datasource. | +| `polaris.persistence.relational.jdbc.driver-directory` | | `string` | Optional directory of JDBC driver jars loaded for the Polaris-managed datasource. When unset, Polaris looks for `jdbc-drivers` beside `quarkus-run.jar` in binary distributions. | +| `polaris.persistence.relational.jdbc.username` | | `string` | Optional username for the Polaris-managed datasource. | +| `polaris.persistence.relational.jdbc.password` | | `string` | Optional password for the Polaris-managed datasource. | +| `polaris.persistence.relational.jdbc.maximum-pool-size` | | `int` | Optional maximum connection pool size for the Polaris-managed datasource. | +| `polaris.persistence.relational.jdbc.minimum-idle` | | `int` | Optional minimum idle connection count for the Polaris-managed datasource. | +| `polaris.persistence.relational.jdbc.connection-timeout-in-ms` | | `long` | Optional connection timeout in milliseconds for the Polaris-managed datasource. | | `polaris.persistence.relational.jdbc.max-retries` | | `int` | | | `polaris.persistence.relational.jdbc.max-duration-in-ms` | | `long` | | | `polaris.persistence.relational.jdbc.initial-delay-in-ms` | | `long` | | | `polaris.persistence.relational.jdbc.database-type` | | `string` | | -| `polaris.persistence.relational.jdbc.jdbc-url` | | `string` | Optional JDBC URL for a Polaris-managed datasource. When set, Polaris creates the JDBC pool directly instead of using the Quarkus datasource extension. | -| `polaris.persistence.relational.jdbc.driver` | | `string` | Optional JDBC driver class name for the Polaris-managed datasource, for example `org.postgresql.Driver`. | -| `polaris.persistence.relational.jdbc.driver-directory` | | `string` | Optional directory of JDBC driver jars loaded for the Polaris-managed datasource. When unset, Polaris looks for `jdbc-drivers` beside `quarkus-run.jar` in binary distributions. | -| `polaris.persistence.relational.jdbc.username` | | `string` | Optional username for the Polaris-managed datasource. | -| `polaris.persistence.relational.jdbc.password` | | `string` | Optional password for the Polaris-managed datasource. | -| `polaris.persistence.relational.jdbc.maximum-pool-size` | | `int` | Optional maximum connection pool size for the Polaris-managed datasource. | -| `polaris.persistence.relational.jdbc.minimum-idle` | | `int` | Optional minimum idle connection count for the Polaris-managed datasource. | -| `polaris.persistence.relational.jdbc.connection-timeout-in-ms` | | `long` | Optional connection timeout in milliseconds for the Polaris-managed datasource. |