diff --git a/CHANGELOG.md b/CHANGELOG.md index 7abb675ad4f..8d1ed7a9d92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,7 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti - Added support for multiple event listeners. Set `polaris.event-listener.types` to a comma-separated list of event listener types to enable multiple event listeners. - Added support for enabling only a subset of event types and event categories per event listener. Set `polaris.event-listener.`_``_`.enabled-event-types` or `polaris.event-listener.`_``_`.enabled-event-categories` to the list of event types or categories for the specified event listener to only consume the selected subset of events. - Added support for **Apache Ranger** as an external authorizer (Beta). +- Added source-level support for building a MySQL-capable Polaris server for the relational JDBC backend. Official Polaris release artifacts do not include the MySQL JDBC driver, which is GPL-licensed. Users who need this path should download the official Polaris source release and build their own derivative from that source tree (`./gradlew :polaris-server:assemble -PincludeMysqlDriver=true`); see `runtime/server/README.md` in the unpacked source release for build details. ### Changes - Improved Python CLI error messages and exit codes for invalid arguments and configuration errors. diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index de1bdb25500..875f579ebf9 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -137,6 +137,9 @@ tasks.register("verifyBomDependencies") { ":aggregated-license-report", ":polaris-config-docs-site", ":polaris-distribution", + // Integration-test-only module that pulls the GPL MySQL JDBC driver; it is not a + // published library, so it must not be referenced from the BOM. + ":polaris-relational-jdbc-mysql-tests", ) ) } diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 89bd17757a4..07222129263 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -33,6 +33,7 @@ polaris-admin=runtime/admin polaris-runtime-common=runtime/common polaris-runtime-test-common=runtime/test-common polaris-relational-jdbc=persistence/relational-jdbc +polaris-relational-jdbc-mysql-tests=persistence/relational-jdbc-mysql/tests polaris-tests=integration-tests aggregated-license-report=aggregated-license-report polaris-immutables=tools/immutables diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java index 8505d34249e..b5269620ef8 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java @@ -325,6 +325,55 @@ public void testCreateCatalogWithGcpStorageConfig() { } } + @Test + public void testCatalogsDifferingOnlyByCaseCoexist() { + // Cross-backend regression check that identifier uniqueness is case-sensitive on the + // `entities` table. Notably this catches MySQL collation regressions: a default + // case-insensitive collation would collapse `foo` and `Foo` into a single row via the + // (realm_id, catalog_id, parent_id, type_code, name) unique constraint, diverging from + // PostgreSQL's case-sensitive TEXT semantics. + AwsStorageConfigInfo awsConfigModel = + AwsStorageConfigInfo.builder() + .setRoleArn("arn:aws:iam::000000000000:role/polaris-it") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://test-bucket/")) + .build(); + String lowerName = client.newEntityName("foo"); + String upperName = lowerName.replaceFirst("foo", "Foo"); + Catalog lowerCatalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(lowerName) + .setProperties(new CatalogProperties("s3://test-bucket/" + lowerName)) + .setStorageConfigInfo(awsConfigModel) + .build(); + Catalog upperCatalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(upperName) + .setProperties(new CatalogProperties("s3://test-bucket/" + upperName)) + .setStorageConfigInfo(awsConfigModel) + .build(); + managementApi.createCatalog(lowerCatalog); + try { + managementApi.createCatalog(upperCatalog); + try { + try (Response r = managementApi.request("v1/catalogs/" + lowerName).get()) { + assertThat(r).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + assertThat(r.readEntity(Catalog.class).getName()).isEqualTo(lowerName); + } + try (Response r = managementApi.request("v1/catalogs/" + upperName).get()) { + assertThat(r).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + assertThat(r.readEntity(Catalog.class).getName()).isEqualTo(upperName); + } + } finally { + managementApi.deleteCatalog(upperName); + } + } finally { + managementApi.deleteCatalog(lowerName); + } + } + @Test public void testCreateCatalogWithNullBaseLocation() { AwsStorageConfigInfo awsConfigModel = diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisPolicyServiceIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisPolicyServiceIntegrationTest.java index 1389d019a87..dade6f84d42 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisPolicyServiceIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisPolicyServiceIntegrationTest.java @@ -474,6 +474,47 @@ public void testDropPolicy() { .hasSize(0); } + @Test + public void testAttachAndDetachPolicyWithNonTrivialJsonParameters() { + // Regression coverage for `JdbcBasePersistenceImpl.deleteFromPolicyMappingRecords`: + // the DELETE WHERE clause passes the `parameters` JSON column value verbatim and + // must match the stored row. On MySQL this requires `CAST(? AS JSON)` placeholder + // selection (via `ModelRegistry.isJsonColumn` + `DatabaseType.asJsonConditionPlaceholder`); + // on PostgreSQL the `parameters` column is jsonb and matches with a plain `?` + // placeholder bound to a `PGobject(jsonb)` value. Either way, attaching with + // non-trivial JSON parameters and then detaching must succeed. + restCatalog.createNamespace(NS1); + policyApi.createPolicy( + currentCatalogName, + NS1_P1, + PredefinedPolicyTypes.DATA_COMPACTION, + EXAMPLE_TABLE_MAINTENANCE_POLICY_CONTENT, + "test policy"); + + PolicyAttachmentTarget catalogTarget = + PolicyAttachmentTarget.builder().setType(PolicyAttachmentTarget.TypeEnum.CATALOG).build(); + Map nonTrivialParameters = + Map.of("retention", "30days", "scope", "namespace-and-tables", "owner", "polaris-it"); + policyApi.attachPolicy(currentCatalogName, NS1_P1, catalogTarget, nonTrivialParameters); + + // Detach exercises `deleteFromPolicyMappingRecords` which uses the `parameters` + // JSON column in the DELETE WHERE clause. A regression that drops the JSON + // placeholder selection (or that re-introduces `Converter.MysqlJsonValue`-based + // dispatch in the wrong way) would silently fail to delete the row on MySQL. + policyApi.detachPolicy(currentCatalogName, NS1_P1, catalogTarget); + + // Re-attaching the same target with different non-trivial parameters must succeed + // (the previous row must have actually been deleted, not just orphaned). + policyApi.attachPolicy( + currentCatalogName, + NS1_P1, + catalogTarget, + Map.of("retention", "7days", "scope", "namespace-only")); + policyApi.detachPolicy(currentCatalogName, NS1_P1, catalogTarget); + + policyApi.dropPolicy(currentCatalogName, NS1_P1); + } + @Test public void testDropNonExistingPolicy() { restCatalog.createNamespace(NS1); diff --git a/persistence/relational-jdbc-mysql/tests/build.gradle.kts b/persistence/relational-jdbc-mysql/tests/build.gradle.kts new file mode 100644 index 00000000000..b5944ae02f1 --- /dev/null +++ b/persistence/relational-jdbc-mysql/tests/build.gradle.kts @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + alias(libs.plugins.quarkus) + id("org.kordamp.gradle.jandex") + id("polaris-runtime") +} + +// Mirror `runtime/test-common`'s antlr/Scala exclusions and additionally drop +// jackson-module-scala: Jackson's ServiceLoader otherwise tries to instantiate +// DefaultScalaModule during RESTEasy provider init and fails. +configurations.all { + if (name != "checkstyle") { + exclude(group = "org.antlr", module = "antlr4-runtime") + exclude(group = "org.scala-lang", module = "scala-library") + exclude(group = "org.scala-lang", module = "scala-reflect") + exclude(group = "com.fasterxml.jackson.module", module = "jackson-module-scala_2.12") + exclude(group = "com.fasterxml.jackson.module", module = "jackson-module-scala_2.13") + } +} + +dependencies { + implementation(platform(libs.quarkus.bom)) + implementation("io.quarkus:quarkus-rest-jackson") + // The MySQL JDBC driver lives only in this test module so it stays off the + // production-facing `runtime/service` classpath. + implementation("io.quarkus:quarkus-jdbc-mysql") + implementation(project(":polaris-relational-jdbc")) + implementation(project(":polaris-runtime-service")) + + testImplementation(project(":polaris-runtime-test-common")) + + intTestImplementation("io.quarkus:quarkus-junit") + intTestImplementation("io.rest-assured:rest-assured") + intTestImplementation(project(":polaris-api-management-model")) + intTestImplementation(project(":polaris-tests")) + // Reuse the existing `ServerManager` (the PolarisServerManager SPI impl) from + // runtime-service test fixtures instead of duplicating it in this module. + intTestImplementation(testFixtures(project(":polaris-runtime-service"))) + intTestImplementation(platform(libs.iceberg.bom)) + intTestImplementation("org.apache.iceberg:iceberg-api") + intTestImplementation("org.apache.iceberg:iceberg-core") + // CatalogTests / ViewCatalogTests test fixtures used by Mysql*IT base classes. + intTestImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests") + intTestImplementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests") + + intTestImplementation(platform(libs.testcontainers.bom)) + intTestImplementation("org.testcontainers:testcontainers-junit-jupiter") + intTestImplementation("org.testcontainers:testcontainers-mysql") + intTestImplementation(project(":polaris-container-spec-helper")) + + // RESTEasy Classic (via keycloak-admin-client) provides the jakarta.ws.rs.client.ClientBuilder + // SPI used by `PolarisClient`. Must remain `intTestRuntimeOnly` so it does not participate + // in Quarkus augmentation (which uses RESTEasy Reactive). + intTestRuntimeOnly(libs.keycloak.admin.client) +} + +// `runtime/defaults` keeps the MySQL named datasource off by default; flip it on at Quarkus +// build time for this test module only. The matching `active=true` runtime override is set +// by `MysqlRelationalJdbcLifeCycleManagement`. +quarkus { quarkusBuildProperties.put("quarkus.datasource.mysql.jdbc", "true") } + +tasks.named("javadoc") { dependsOn("jandex") } + +tasks.withType { + if (System.getenv("AWS_REGION") == null) { + environment("AWS_REGION", "us-west-2") + } + environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") + val apiVersion = System.getenv("DOCKER_API_VERSION") ?: "1.44" + systemProperty("api.version", apiVersion) + jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") + systemProperty("java.security.manager", "allow") + maxParallelForks = 1 + + val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") + jvmArgumentProviders.add( + CommandLineArgumentProvider { + listOf("-Dquarkus.log.file.path=${logsDir.resolve("polaris.log").absolutePath}") + } + ) + doFirst { + logsDir.deleteRecursively() + project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete() + } +} diff --git a/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlApplicationIT.java b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlApplicationIT.java new file mode 100644 index 00000000000..01d35b35605 --- /dev/null +++ b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlApplicationIT.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.mysql.tests; + +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest; + +@TestProfile(MysqlRelationalJdbcProfile.class) +@QuarkusIntegrationTest +public class MysqlApplicationIT extends PolarisApplicationIntegrationTest {} diff --git a/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlManagementServiceIT.java b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlManagementServiceIT.java new file mode 100644 index 00000000000..10f2dc35e80 --- /dev/null +++ b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlManagementServiceIT.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.mysql.tests; + +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisManagementServiceIntegrationTest; + +@TestProfile(MysqlRelationalJdbcProfile.class) +@QuarkusIntegrationTest +public class MysqlManagementServiceIT extends PolarisManagementServiceIntegrationTest {} diff --git a/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlPolicyServiceIT.java b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlPolicyServiceIT.java new file mode 100644 index 00000000000..b0104d8908b --- /dev/null +++ b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlPolicyServiceIT.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.mysql.tests; + +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisPolicyServiceIntegrationTest; + +@TestProfile(MysqlRelationalJdbcProfile.class) +@QuarkusIntegrationTest +public class MysqlPolicyServiceIT extends PolarisPolicyServiceIntegrationTest {} diff --git a/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlRelationalJdbcLifeCycleManagement.java b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlRelationalJdbcLifeCycleManagement.java new file mode 100644 index 00000000000..f21b0a414d5 --- /dev/null +++ b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlRelationalJdbcLifeCycleManagement.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.mysql.tests; + +import static org.apache.polaris.containerspec.ContainerSpecHelper.containerSpecHelper; + +import io.quarkus.test.common.DevServicesContext; +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import java.util.Map; +import org.testcontainers.containers.MySQLContainer; + +public class MysqlRelationalJdbcLifeCycleManagement + implements QuarkusTestResourceLifecycleManager, DevServicesContext.ContextAware { + public static final String INIT_SCRIPT = "init-script"; + + private MySQLContainer mysql; + private String initScript; + private DevServicesContext context; + + @Override + public void init(Map initArgs) { + initScript = initArgs.get(INIT_SCRIPT); + } + + @Override + @SuppressWarnings("resource") + public Map start() { + mysql = + new MySQLContainer<>( + containerSpecHelper("mysql", MysqlRelationalJdbcLifeCycleManagement.class) + .dockerImageName(null) + .asCompatibleSubstituteFor("mysql")) + .withDatabaseName("POLARIS_SCHEMA"); + + if (initScript != null) { + mysql.withInitScript(initScript); + } + + context.containerNetworkId().ifPresent(mysql::withNetworkMode); + mysql.start(); + + return Map.of( + "polaris.persistence.type", + "relational-jdbc", + "polaris.persistence.relational.jdbc.database-type", + "mysql", + "polaris.persistence.relational.jdbc.max-retries", + "2", + // The named MySQL datasource is declared as inactive in the bundled defaults; tests + // (and production users) opt in by setting `active=true` on the named datasource. + "quarkus.datasource.mysql.active", + "true", + "quarkus.datasource.mysql.jdbc.url", + mysql.getJdbcUrl(), + "quarkus.datasource.mysql.username", + mysql.getUsername(), + "quarkus.datasource.mysql.password", + mysql.getPassword(), + "quarkus.datasource.mysql.jdbc.initial-size", + "10"); + } + + @Override + public void stop() { + if (mysql != null) { + try { + mysql.stop(); + } finally { + mysql = null; + } + } + } + + @Override + public void setIntegrationTestContext(DevServicesContext context) { + this.context = context; + } +} diff --git a/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlRelationalJdbcProfile.java b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlRelationalJdbcProfile.java new file mode 100644 index 00000000000..c79f5779511 --- /dev/null +++ b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlRelationalJdbcProfile.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.mysql.tests; + +import io.quarkus.test.junit.QuarkusTestProfile; +import java.util.List; +import java.util.Map; + +public class MysqlRelationalJdbcProfile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of( + "polaris.persistence.auto-bootstrap-types", + "relational-jdbc", + "polaris.persistence.relational.jdbc.database-type", + "mysql"); + } + + @Override + public List testResources() { + return List.of( + new QuarkusTestProfile.TestResourceEntry( + MysqlRelationalJdbcLifeCycleManagement.class, Map.of())); + } +} diff --git a/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlRestCatalogIT.java b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlRestCatalogIT.java new file mode 100644 index 00000000000..7d120850820 --- /dev/null +++ b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlRestCatalogIT.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.mysql.tests; + +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisRestCatalogFileIntegrationTest; + +@TestProfile(MysqlRelationalJdbcProfile.class) +@QuarkusIntegrationTest +public class MysqlRestCatalogIT extends PolarisRestCatalogFileIntegrationTest {} diff --git a/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlViewFileIT.java b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlViewFileIT.java new file mode 100644 index 00000000000..d9bfdc324b4 --- /dev/null +++ b/persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlViewFileIT.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.mysql.tests; + +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.quarkus.test.junit.TestProfile; +import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTestBase; + +@TestProfile(MysqlRelationalJdbcProfile.class) +@QuarkusIntegrationTest +public class MysqlViewFileIT extends PolarisRestCatalogViewFileIntegrationTestBase {} diff --git a/persistence/relational-jdbc-mysql/tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/persistence/relational-jdbc-mysql/tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 00000000000..b3dd7d7c069 --- /dev/null +++ b/persistence/relational-jdbc-mysql/tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.it.ServerManager diff --git a/persistence/relational-jdbc-mysql/tests/src/intTest/resources/org/apache/polaris/persistence/relational/jdbc/mysql/tests/Dockerfile-mysql-version b/persistence/relational-jdbc-mysql/tests/src/intTest/resources/org/apache/polaris/persistence/relational/jdbc/mysql/tests/Dockerfile-mysql-version new file mode 100644 index 00000000000..4f06edb681e --- /dev/null +++ b/persistence/relational-jdbc-mysql/tests/src/intTest/resources/org/apache/polaris/persistence/relational/jdbc/mysql/tests/Dockerfile-mysql-version @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Dockerfile to provide the image name and tag to a test. +# Version is managed by Renovate - do not edit. +FROM mysql:8.0 diff --git a/persistence/relational-jdbc/MYSQL.md b/persistence/relational-jdbc/MYSQL.md new file mode 100644 index 00000000000..16d4b898499 --- /dev/null +++ b/persistence/relational-jdbc/MYSQL.md @@ -0,0 +1,84 @@ + + +# MySQL support for the Relational JDBC backend (custom downstream build) + +The Relational JDBC persistence backend can run on MySQL 8.0+, alongside the officially supported PostgreSQL, CockroachDB and H2 options. + +> **The MySQL JDBC driver is GPL-licensed (ASF Category X) and is intentionally not bundled in the official Apache Polaris release artifacts** (see [issue #2491](https://github.com/apache/polaris/issues/2491)). The Java code that supports MySQL ships in the official source release, but enabling it requires building your own runner from source with the opt-in flag described below. The resulting runner is a *custom downstream derivative*, not an official Apache Polaris artifact, and the builder is responsible for satisfying the driver's GPL v2 + FOSS Exception obligations. + +## Building a MySQL-capable server + +Add the MySQL JDBC driver to the runner with the `-PincludeMysqlDriver=true` Gradle property: + +```shell +./gradlew :polaris-server:assemble -PincludeMysqlDriver=true +``` + +The Docker image can be built the same way with `-Dquarkus.container-image.build=true`. To avoid overwriting the GPL-free image, pass a distinct image name: + +```shell +./gradlew \ + :polaris-server:assemble \ + :polaris-server:quarkusAppPartsBuild --rerun \ + -PincludeMysqlDriver=true \ + -Dquarkus.container-image.build=true \ + -Dquarkus.container-image.name=polaris-mysql +``` + +### Verification contract under `-PincludeMysqlDriver=true` + +Supported tasks: `:polaris-server:assemble` and the Quarkus image build (`quarkusAppPartsBuild` with `-Dquarkus.container-image.build=true`). + +`check`, `build`, `publishToMavenLocal`, and the `generateLicenseReport` / `checkLicense` / `licenseReportZip` tasks will fail by design because the GPL MySQL JDBC driver is intentionally absent from `LICENSE`. No license tasks are disabled — the default (GPL-free) build continues to run and pass all license checks unchanged. Downstream builders distributing the resulting (non-official) artifact are responsible for satisfying the driver's GPL v2 + FOSS Exception obligations. + +## Runtime configuration + +MySQL is exposed as a Quarkus *named* datasource (`mysql`) that is declared inactive by default. To use the MySQL backend at runtime, set the following properties (or the equivalent environment variables, e.g. `QUARKUS_DATASOURCE_MYSQL_JDBC_URL`): + +```properties +polaris.persistence.type=relational-jdbc +polaris.persistence.relational.jdbc.database-type=mysql + +quarkus.datasource.mysql.active=true +quarkus.datasource.mysql.jdbc.url=jdbc:mysql://:3306/POLARIS_SCHEMA +quarkus.datasource.mysql.username= +quarkus.datasource.mysql.password= +``` + +PostgreSQL, CockroachDB and H2 continue to use the default (unnamed) datasource and require no changes to existing configuration. + +If `database-type=mysql` is configured but the `mysql` named datasource is not available (for example, the runner was built without `-PincludeMysqlDriver=true`, or `quarkus.datasource.mysql.active` is left at its default of `false`), the server fails fast at startup with a message pointing back to the build flag and the `active` / connection properties above, rather than silently falling back to the default datasource. + +## Bootstrapping the MySQL backend + +The Polaris admin tool does not currently bundle the MySQL JDBC driver, so the standard admin-tool `bootstrap` flow does not apply. Instead, configure the Polaris server to bootstrap itself on first startup: + +```properties +polaris.persistence.auto-bootstrap-types=in-memory,relational-jdbc +polaris.realm-context.realms= +``` + +```shell +POLARIS_BOOTSTRAP_CREDENTIALS=,, +``` + +The server will execute `mysql/schema-v4.sql` and create the root principal automatically. Run a single replica for the initial bootstrap to avoid races; the bootstrap log line confirms completion. + +**Auto-bootstrap on persistent backends is not yet idempotent** ([apache/polaris#2324](https://github.com/apache/polaris/issues/2324)): re-running the server with these settings against an already-bootstrapped database fails on startup with `metastore manager has already been bootstrapped`. Treat the auto-bootstrap startup as a one-time operation until #2324 is resolved. diff --git a/persistence/relational-jdbc/build.gradle.kts b/persistence/relational-jdbc/build.gradle.kts index d0a3c15cfcc..ba8bae03c1c 100644 --- a/persistence/relational-jdbc/build.gradle.kts +++ b/persistence/relational-jdbc/build.gradle.kts @@ -37,6 +37,8 @@ dependencies { compileOnly("io.opentelemetry:opentelemetry-api") implementation(libs.smallrye.common.annotation) // @Identifier + compileOnly(platform(libs.quarkus.bom)) + compileOnly("io.quarkus:quarkus-agroal") // DataSourceLiteral for named DataSource lookup implementation(libs.postgresql) compileOnly(project(":polaris-immutables")) diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java index cc104e1f39f..e53003b4d3d 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java @@ -34,7 +34,8 @@ public enum DatabaseType { POSTGRES("postgres"), COCKROACHDB("cockroachdb"), - H2("h2"); + H2("h2"), + MYSQL("mysql"); private final String displayName; // Store the user-friendly name @@ -47,6 +48,22 @@ public String getDisplayName() { return displayName; } + /** + * Returns the SQL placeholder text to use for equality comparisons against a JSON-typed column. + * + *

MySQL JSON columns require {@code CAST(? AS JSON)} so the right-hand side is parsed into a + * JSON value and compared structurally; comparing a JSON column to a plain string parameter is + * formatting-sensitive (e.g. {@code '{"a":1}'} vs {@code '{"a": 1}'}) and may not match. + * + *

All other supported databases use the default {@code ?} placeholder. + */ + public String asJsonConditionPlaceholder() { + return switch (this) { + case MYSQL -> "CAST(? AS JSON)"; + case POSTGRES, COCKROACHDB, H2 -> "?"; + }; + } + /** * Returns the latest schema version available for this database type. This is used as the default * schema version for new installations. @@ -56,6 +73,7 @@ public int getLatestSchemaVersion() { case POSTGRES -> 4; // PostgreSQL has schemas v1, v2, v3, v4 case COCKROACHDB -> 4; // CockroachDB schema version kept in sync with PostgreSQL case H2 -> 4; // H2 uses same schemas as PostgreSQL + case MYSQL -> 4; // MySQL has schema v4 only }; } @@ -64,6 +82,7 @@ public static DatabaseType fromDisplayName(String displayName) { case "h2" -> DatabaseType.H2; case "postgresql" -> DatabaseType.POSTGRES; case "cockroachdb" -> DatabaseType.COCKROACHDB; + case "mysql" -> DatabaseType.MYSQL; default -> throw new IllegalStateException("Unsupported DatabaseType: '" + displayName + "'"); }; } @@ -100,6 +119,8 @@ public static DatabaseType inferFromConnection( inferredType = DatabaseType.POSTGRES; } else if (productName.contains("h2")) { inferredType = DatabaseType.H2; + } else if (productName.contains("mysql")) { + inferredType = DatabaseType.MYSQL; } // If a type was explicitly configured, use it and validate diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java index 372ccfe1ca7..2ade96b51c3 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatasourceOperations.java @@ -64,6 +64,14 @@ public class DatasourceOperations { private static final String H2_SCHEMA_DOES_NOT_EXIST = "90079"; private static final String H2_TABLE_DOES_NOT_EXIST = "42S02"; + // MYSQL STATUS CODES + // SQLSTATE 23000 = integrity constraint violation (duplicate key / FK / NOT NULL / CHECK). + // Pair with vendor error 1062 (duplicate entry) to narrow down to duplicate-key only. + private static final String MYSQL_CONSTRAINT_VIOLATION = "23000"; + private static final int MYSQL_DUPLICATE_ENTRY_CODE = 1062; + // SQLSTATE 42S02 = base table or view not found (shared with H2). + private static final String MYSQL_TABLE_DOES_NOT_EXIST = "42S02"; + // POSTGRES RETRYABLE EXCEPTIONS private static final String SERIALIZATION_FAILURE_SQL_CODE = "40001"; @@ -94,7 +102,7 @@ public DatasourceOperations( } } - DatabaseType getDatabaseType() { + public DatabaseType getDatabaseType() { return databaseType; } @@ -450,7 +458,17 @@ public interface TransactionCallback { } public boolean isUniquenessConstraintViolation(SQLException e) { - return UNIQUENESS_CONSTRAINT_VIOLATION_SQL_CODE.equals(e.getSQLState()); + // PostgreSQL / CockroachDB: SQLSTATE 23505 is specifically unique_violation. + if (UNIQUENESS_CONSTRAINT_VIOLATION_SQL_CODE.equals(e.getSQLState())) { + return true; + } + // MySQL: combine SQLSTATE 23000 with vendor error 1062 to match PG's 23505 precision. + if (databaseType == DatabaseType.MYSQL + && MYSQL_CONSTRAINT_VIOLATION.equals(e.getSQLState()) + && e.getErrorCode() == MYSQL_DUPLICATE_ENTRY_CODE) { + return true; + } + return false; } public boolean isRelationDoesNotExist(SQLException e) { @@ -458,7 +476,9 @@ public boolean isRelationDoesNotExist(SQLException e) { && (databaseType == DatabaseType.POSTGRES || databaseType == DatabaseType.COCKROACHDB)) || ((H2_SCHEMA_DOES_NOT_EXIST.equals(e.getSQLState()) || H2_TABLE_DOES_NOT_EXIST.equals(e.getSQLState())) - && databaseType == DatabaseType.H2); + && databaseType == DatabaseType.H2) + || (MYSQL_TABLE_DOES_NOT_EXIST.equals(e.getSQLState()) + && databaseType == DatabaseType.MYSQL); } private Connection borrowConnection() throws SQLException { diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java index cc711cc646a..4ed42ae1caa 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcBasePersistenceImpl.java @@ -90,6 +90,9 @@ public class JdbcBasePersistenceImpl private final PolarisDiagnostics diagnostics; private final DatasourceOperations datasourceOperations; + // Bound to the active DatabaseType at construction time so call sites do not need to thread + // the database type through every QueryGenerator.generate*Query(...) invocation. + private final QueryGenerator queryGenerator; private final PrincipalSecretsGenerator secretsGenerator; private final String realmId; private final int schemaVersion; @@ -105,6 +108,7 @@ public JdbcBasePersistenceImpl( int schemaVersion) { this.diagnostics = diagnostics; this.datasourceOperations = databaseOperations; + this.queryGenerator = new QueryGenerator(databaseOperations.getDatabaseType()); this.secretsGenerator = secretsGenerator; this.realmId = realmId; this.schemaVersion = schemaVersion; @@ -232,7 +236,7 @@ private void persistEntity( int rowsUpdated = queryAction.apply( connection, - QueryGenerator.generateUpdateQuery( + queryGenerator.generateUpdateQuery( ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, values, @@ -342,7 +346,7 @@ public void deleteEntity(@NonNull PolarisCallContext callCtx, @NonNull PolarisBa realmId); try { datasourceOperations.executeUpdate( - QueryGenerator.generateDeleteQuery( + queryGenerator.generateDeleteQuery( ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, params)); } catch (SQLException e) { throw new RuntimeException( @@ -359,7 +363,7 @@ public void deleteFromGrantRecords( modelGrantRecord.toMap(datasourceOperations.getDatabaseType()); whereClause.put("realm_id", realmId); datasourceOperations.executeUpdate( - QueryGenerator.generateDeleteQuery( + queryGenerator.generateDeleteQuery( ModelGrantRecord.ALL_COLUMNS, ModelGrantRecord.TABLE_NAME, whereClause)); } catch (SQLException e) { throw new RuntimeException( @@ -390,21 +394,21 @@ public void deleteAll(@NonNull PolarisCallContext callCtx) { connection -> { datasourceOperations.execute( connection, - QueryGenerator.generateDeleteQuery( + queryGenerator.generateDeleteQuery( ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, params)); datasourceOperations.execute( connection, - QueryGenerator.generateDeleteQuery( + queryGenerator.generateDeleteQuery( ModelGrantRecord.ALL_COLUMNS, ModelGrantRecord.TABLE_NAME, params)); datasourceOperations.execute( connection, - QueryGenerator.generateDeleteQuery( + queryGenerator.generateDeleteQuery( ModelPrincipalAuthenticationData.ALL_COLUMNS, ModelPrincipalAuthenticationData.TABLE_NAME, params)); datasourceOperations.execute( connection, - QueryGenerator.generateDeleteQuery( + queryGenerator.generateDeleteQuery( ModelPolicyMappingRecord.ALL_COLUMNS, ModelPolicyMappingRecord.TABLE_NAME, params)); @@ -422,7 +426,7 @@ public PolarisBaseEntity lookupEntity( Map params = Map.of("catalog_id", catalogId, "id", entityId, "type_code", typeCode, "realm_id", realmId); return getPolarisBaseEntity( - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, params)); } @@ -446,7 +450,7 @@ public PolarisBaseEntity lookupEntityByName( "realm_id", realmId); return getPolarisBaseEntity( - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, params)); } @@ -552,7 +556,7 @@ private PreparedQuery buildEntityQuery( whereGreater = Map.of(); } - return QueryGenerator.generateSelectQuery( + return queryGenerator.generateSelectQuery( queryProjections, ModelEntity.TABLE_NAME, whereEquals, whereGreater, orderByColumnName); } @@ -632,7 +636,7 @@ public int lookupEntityGrantRecordsVersion( Map.of("catalog_id", catalogId, "id", entityId, "realm_id", realmId); PolarisBaseEntity b = getPolarisBaseEntity( - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, params)); return b == null ? 0 : b.getGrantRecordsVersion(); } @@ -662,7 +666,7 @@ public PolarisGrantRecord lookupGrantRecord( try { var results = datasourceOperations.executeSelect( - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelGrantRecord.ALL_COLUMNS, ModelGrantRecord.TABLE_NAME, params), new ModelGrantRecord()); if (results.size() > 1) { @@ -694,7 +698,7 @@ public List loadAllGrantRecordsOnSecurable( try { var results = datasourceOperations.executeSelect( - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelGrantRecord.ALL_COLUMNS, ModelGrantRecord.TABLE_NAME, params), new ModelGrantRecord()); return results == null ? Collections.emptyList() : results; @@ -717,7 +721,7 @@ public List loadAllGrantRecordsOnGrantee( try { var results = datasourceOperations.executeSelect( - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelGrantRecord.ALL_COLUMNS, ModelGrantRecord.TABLE_NAME, params), new ModelGrantRecord()); return results == null ? Collections.emptyList() : results; @@ -746,7 +750,7 @@ public boolean hasChildren( try { var results = datasourceOperations.executeSelect( - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, params), new ModelEntity(schemaVersion)); return results != null && !results.isEmpty(); @@ -851,7 +855,7 @@ public PolarisPrincipalSecrets loadPrincipalSecrets( try { var results = datasourceOperations.executeSelect( - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelPrincipalAuthenticationData.ALL_COLUMNS, ModelPrincipalAuthenticationData.TABLE_NAME, params), @@ -991,7 +995,7 @@ public PolarisPrincipalSecrets rotatePrincipalSecrets( ModelPrincipalAuthenticationData modelPrincipalAuthenticationData = ModelPrincipalAuthenticationData.fromPrincipalAuthenticationData(principalSecrets); datasourceOperations.executeUpdate( - QueryGenerator.generateUpdateQuery( + queryGenerator.generateUpdateQuery( ModelPrincipalAuthenticationData.ALL_COLUMNS, ModelPrincipalAuthenticationData.TABLE_NAME, modelPrincipalAuthenticationData @@ -1021,7 +1025,7 @@ public void deletePrincipalSecrets( Map.of("principal_client_id", clientId, "principal_id", principalId, "realm_id", realmId); try { datasourceOperations.executeUpdate( - QueryGenerator.generateDeleteQuery( + queryGenerator.generateDeleteQuery( ModelPrincipalAuthenticationData.ALL_COLUMNS, ModelPrincipalAuthenticationData.TABLE_NAME, params)); @@ -1113,7 +1117,7 @@ private boolean handleInheritablePolicy( ModelPolicyMappingRecord modelPolicyMappingRecord = ModelPolicyMappingRecord.fromPolicyMappingRecord(record); PreparedQuery updateQuery = - QueryGenerator.generateUpdateQuery( + queryGenerator.generateUpdateQuery( ModelPolicyMappingRecord.ALL_COLUMNS, ModelPolicyMappingRecord.TABLE_NAME, modelPolicyMappingRecord @@ -1139,7 +1143,7 @@ public void deleteFromPolicyMappingRecords( modelPolicyMappingRecord.toMap(datasourceOperations.getDatabaseType()); objectMap.put("realm_id", realmId); datasourceOperations.executeUpdate( - QueryGenerator.generateDeleteQuery( + queryGenerator.generateDeleteQuery( ModelPolicyMappingRecord.ALL_COLUMNS, ModelPolicyMappingRecord.TABLE_NAME, objectMap)); @@ -1168,7 +1172,7 @@ public void deleteAllEntityPolicyMappingRecords( } queryParams.put("realm_id", realmId); datasourceOperations.executeUpdate( - QueryGenerator.generateDeleteQuery( + queryGenerator.generateDeleteQuery( ModelPolicyMappingRecord.ALL_COLUMNS, ModelPolicyMappingRecord.TABLE_NAME, queryParams)); @@ -1203,7 +1207,7 @@ public PolarisPolicyMappingRecord lookupPolicyMappingRecord( realmId); List results = fetchPolicyMappingRecords( - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelPolicyMappingRecord.ALL_COLUMNS, ModelPolicyMappingRecord.TABLE_NAME, params)); Preconditions.checkState(results.size() <= 1, "More than one policy mapping records found"); return results.size() == 1 ? results.getFirst() : null; @@ -1227,7 +1231,7 @@ public List loadPoliciesOnTargetByType( "realm_id", realmId); return fetchPolicyMappingRecords( - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelPolicyMappingRecord.ALL_COLUMNS, ModelPolicyMappingRecord.TABLE_NAME, params)); } @@ -1264,7 +1268,7 @@ private List fetchPolicyMappingRecords( "realm_id", realmId); return fetchPolicyMappingRecords( - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelPolicyMappingRecord.ALL_COLUMNS, ModelPolicyMappingRecord.TABLE_NAME, params), connection); } @@ -1276,7 +1280,7 @@ public List loadAllPoliciesOnTarget( Map params = Map.of("target_catalog_id", targetCatalogId, "target_id", targetId, "realm_id", realmId); return fetchPolicyMappingRecords( - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelPolicyMappingRecord.ALL_COLUMNS, ModelPolicyMappingRecord.TABLE_NAME, params)); } @@ -1298,7 +1302,7 @@ public List loadAllTargetsOnPolicy( "realm_id", realmId); return fetchPolicyMappingRecords( - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelPolicyMappingRecord.ALL_COLUMNS, ModelPolicyMappingRecord.TABLE_NAME, params)); } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index e88aac882af..adbc9d7a395 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -20,8 +20,10 @@ import static org.apache.polaris.core.auth.AuthBootstrapUtil.createPolarisPrincipalForRealm; +import io.quarkus.agroal.DataSource.DataSourceLiteral; import io.smallrye.common.annotation.Identifier; import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Any; import jakarta.enterprise.inject.Instance; import jakarta.enterprise.inject.Produces; import jakarta.inject.Inject; @@ -94,8 +96,38 @@ protected JdbcMetaStoreManagerFactory() {} @Produces @ApplicationScoped static DatasourceOperations produceDatasourceOperations( - Instance dataSource, RelationalJdbcConfiguration relationalJdbcConfiguration) { - return new DatasourceOperations(dataSource.get(), relationalJdbcConfiguration); + Instance defaultDataSource, + @Any Instance dataSources, + RelationalJdbcConfiguration relationalJdbcConfiguration) { + Optional configuredType = relationalJdbcConfiguration.databaseType(); + String mysql = DatabaseType.MYSQL.getDisplayName(); + + // Use a named DataSource if one is declared for the configured database-type; otherwise fall + // back to the default (unnamed) DataSource shared by PostgreSQL, CockroachDB and H2. + // + // MySQL is the only backend that requires its own named DataSource (it needs the GPL MySQL + // driver, which is opt-in at build time and inactive by default). If MySQL is configured but + // its named DataSource is not resolvable, fail fast at the fallback point with actionable + // guidance instead of silently using the default (PostgreSQL-driver) DataSource — that would + // otherwise surface much later as a confusing database-type mismatch. + DataSource ds = + configuredType + .map(type -> dataSources.select(new DataSourceLiteral(type))) + .filter(Instance::isResolvable) + .map(Instance::get) + .orElseGet( + () -> { + if (configuredType.map(type -> type.equals(mysql)).orElse(false)) { + throw new IllegalStateException( + "Database type 'mysql' is configured but the 'mysql' named datasource is" + + " not available. Build the server with -PincludeMysqlDriver=true and" + + " enable the datasource at runtime: quarkus.datasource.mysql.active=" + + "true together with quarkus.datasource.mysql.jdbc.url, .username and" + + " .password."); + } + return defaultDataSource.get(); + }); + return new DatasourceOperations(ds, relationalJdbcConfiguration); } protected PrincipalSecretsGenerator secretsGenerator( diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java index 033835f644c..4037fecd184 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java @@ -32,12 +32,22 @@ import org.apache.polaris.core.storage.StorageLocation; import org.apache.polaris.persistence.relational.jdbc.models.ModelEntity; import org.apache.polaris.persistence.relational.jdbc.models.ModelGrantRecord; +import org.apache.polaris.persistence.relational.jdbc.models.ModelRegistry; import org.jspecify.annotations.NonNull; import org.jspecify.annotations.Nullable; /** - * Utility class to generate parameterized SQL queries (SELECT, INSERT, UPDATE, DELETE). Ensures - * consistent SQL generation and protects against injection by managing parameters separately. + * Generates parameterized SQL queries (SELECT, INSERT, UPDATE, DELETE). Ensures consistent SQL + * generation and protects against injection by managing parameters separately. + * + *

An instance is bound to a {@link DatabaseType} at construction time; the active database is + * not passed through every public method, so callers do not need to thread it through the call + * site. This keeps database-type knowledge contained at the persistence-layer boundary (typically a + * single field per {@code JdbcBasePersistenceImpl}). + * + *

Methods that produce SQL whose shape does not depend on the database type (INSERT, the + * hard-coded grant-record DELETE, the schema-version SELECT, etc.) are kept as {@code static} + * because callers can reuse them without a {@code QueryGenerator} instance. */ public class QueryGenerator { @@ -50,6 +60,17 @@ public record PreparedBatchQuery(String sql, List> parametersList) /** A container for the query fragment SQL string and the ordered parameter values. */ record QueryFragment(String sql, List parameters) {} + private final DatabaseType databaseType; + + public QueryGenerator(@NonNull DatabaseType databaseType) { + this.databaseType = databaseType; + } + + /** Returns the {@link DatabaseType} bound to this generator. */ + public DatabaseType databaseType() { + return databaseType; + } + /** * Generates a SELECT query with projection and filtering. * @@ -59,7 +80,7 @@ record QueryFragment(String sql, List parameters) {} * @return A parameterized SELECT query. * @throws IllegalArgumentException if any whereClause column isn't in projections. */ - public static PreparedQuery generateSelectQuery( + public PreparedQuery generateSelectQuery( @NonNull List projections, @NonNull String tableName, @NonNull Map whereClause) { @@ -75,21 +96,25 @@ public static PreparedQuery generateSelectQuery( * @return A parameterized SELECT query. * @throws IllegalArgumentException if any whereClause column isn't in projections. */ - public static PreparedQuery generateSelectQuery( + public PreparedQuery generateSelectQuery( @NonNull List projections, @NonNull String tableName, @NonNull Map whereEquals, @NonNull Map whereGreater, @Nullable String orderByColumn) { QueryFragment where = - generateWhereClause(new HashSet<>(projections), whereEquals, whereGreater); - PreparedQuery query = generateSelectQuery(projections, tableName, where.sql(), orderByColumn); + generateWhereClause(tableName, new HashSet<>(projections), whereEquals, whereGreater); + PreparedQuery query = + generateSelectQueryStatic(projections, tableName, where.sql(), orderByColumn); return new PreparedQuery(query.sql(), where.parameters()); } /** * Builds a DELETE query to remove grant records for a given entity. * + *

The WHERE clause is hard-coded and does not reference JSON columns, so this method does not + * need a {@link DatabaseType} and stays static. + * * @param entity The target entity (either grantee or securable). * @param realmId The associated realm. * @return A DELETE query removing all grants for this entity. @@ -112,6 +137,9 @@ public static PreparedQuery generateDeleteQueryForEntityGrantRecords( /** * Builds a SELECT query using a list of entity ID pairs (catalog_id, id). * + *

The WHERE clause is constructed manually and does not reference JSON columns, so this method + * stays static. + * * @param realmId Realm to filter by. * @param schemaVersion The schema version of entities table to query * @param entityIds List of PolarisEntityId pairs. @@ -132,7 +160,7 @@ public static PreparedQuery generateSelectQueryWithEntityIds( params.add(realmId); String where = " WHERE (catalog_id, id) IN (" + placeholders + ") AND realm_id = ?"; return new PreparedQuery( - generateSelectQuery( + generateSelectQueryStatic( ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, where, null) .sql(), params); @@ -141,6 +169,9 @@ public static PreparedQuery generateSelectQueryWithEntityIds( /** * Generates an INSERT query for a given table. * + *

INSERT does not have a WHERE clause and the per-column placeholder is always {@code ?}, so + * this method stays static. + * * @param allColumns Columns to insert values into. * @param tableName Target table name. * @param values Values for each column (must match order of columns). @@ -178,13 +209,14 @@ public static PreparedQuery generateInsertQuery( * @param whereClause Conditions for filtering rows to update. * @return UPDATE query with parameter values. */ - public static PreparedQuery generateUpdateQuery( + public PreparedQuery generateUpdateQuery( @NonNull List allColumns, @NonNull String tableName, @NonNull List values, @NonNull Map whereClause) { List bindingParams = new ArrayList<>(values); - QueryFragment where = generateWhereClause(new HashSet<>(allColumns), whereClause, Map.of()); + QueryFragment where = + generateWhereClause(tableName, new HashSet<>(allColumns), whereClause, Map.of()); String setClause = allColumns.stream().map(c -> c + " = ?").collect(Collectors.joining(", ")); String sql = "UPDATE " + getFullyQualifiedTableName(tableName) + " SET " + setClause + where.sql(); @@ -209,7 +241,7 @@ public static PreparedQuery generateUpdateQuery( * @param whereIsNotNull Columns that must be NOT NULL. * @return UPDATE query with parameter bindings. */ - public static PreparedQuery generateUpdateQuery( + public PreparedQuery generateUpdateQuery( @NonNull List tableColumns, @NonNull String tableName, @NonNull Map setClause, @@ -227,7 +259,7 @@ public static PreparedQuery generateUpdateQuery( QueryFragment where = generateWhereClauseExtended( - columns, whereEquals, whereGreater, whereLess, whereIsNull, whereIsNotNull); + tableName, columns, whereEquals, whereGreater, whereLess, whereIsNull, whereIsNotNull); List setParts = new ArrayList<>(); List params = new ArrayList<>(); @@ -254,11 +286,12 @@ public static PreparedQuery generateUpdateQuery( * @param whereClause Column-value filters. * @return DELETE query with parameter bindings. */ - public static PreparedQuery generateDeleteQuery( + public PreparedQuery generateDeleteQuery( @NonNull List tableColumns, @NonNull String tableName, @NonNull Map whereClause) { - QueryFragment where = generateWhereClause(new HashSet<>(tableColumns), whereClause, Map.of()); + QueryFragment where = + generateWhereClause(tableName, new HashSet<>(tableColumns), whereClause, Map.of()); return new PreparedQuery( "DELETE FROM " + getFullyQualifiedTableName(tableName) + where.sql(), where.parameters()); } @@ -267,7 +300,7 @@ public static PreparedQuery generateDeleteQuery( * Builds a DELETE query that supports richer WHERE predicates (equality, greater-than, less-than, * IS NULL, IS NOT NULL). */ - public static PreparedQuery generateDeleteQuery( + public PreparedQuery generateDeleteQuery( @NonNull List tableColumns, @NonNull String tableName, @NonNull Map whereEquals, @@ -278,12 +311,12 @@ public static PreparedQuery generateDeleteQuery( Set columns = new HashSet<>(tableColumns); QueryFragment where = generateWhereClauseExtended( - columns, whereEquals, whereGreater, whereLess, whereIsNull, whereIsNotNull); + tableName, columns, whereEquals, whereGreater, whereLess, whereIsNull, whereIsNotNull); return new PreparedQuery( "DELETE FROM " + getFullyQualifiedTableName(tableName) + where.sql(), where.parameters()); } - private static PreparedQuery generateSelectQuery( + private static PreparedQuery generateSelectQueryStatic( @NonNull List columnNames, @NonNull String tableName, @NonNull String filter, @@ -301,12 +334,13 @@ private static PreparedQuery generateSelectQuery( } @VisibleForTesting - static QueryFragment generateWhereClause( + QueryFragment generateWhereClause( + @NonNull String tableName, @NonNull Set tableColumns, @NonNull Map whereEquals, @NonNull Map whereGreater) { return generateWhereClauseExtended( - tableColumns, whereEquals, whereGreater, Map.of(), Set.of(), Set.of()); + tableName, tableColumns, whereEquals, whereGreater, Map.of(), Set.of(), Set.of()); } private static void validateColumns( @@ -319,7 +353,8 @@ private static void validateColumns( } @VisibleForTesting - static QueryFragment generateWhereClauseExtended( + QueryFragment generateWhereClauseExtended( + @NonNull String tableName, @NonNull Set tableColumns, @NonNull Map whereEquals, @NonNull Map whereGreater, @@ -335,7 +370,12 @@ static QueryFragment generateWhereClauseExtended( List conditions = new ArrayList<>(); List parameters = new ArrayList<>(); for (Map.Entry entry : whereEquals.entrySet()) { - conditions.add(entry.getKey() + " = ?"); + String column = entry.getKey(); + if (ModelRegistry.isJsonColumn(tableName, column)) { + conditions.add(column + " = " + databaseType.asJsonConditionPlaceholder()); + } else { + conditions.add(column + " = ?"); + } parameters.add(entry.getValue()); } for (Map.Entry entry : whereGreater.entrySet()) { @@ -414,7 +454,7 @@ public static PreparedQuery generateOverlapQuery( QueryFragment where = new QueryFragment(clause, finalParams); PreparedQuery query = - generateSelectQuery( + generateSelectQueryStatic( ModelEntity.getAllColumnNames(schemaVersion), ModelEntity.TABLE_NAME, where.sql(), diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java index adfcae5ab1a..5349f9c56cd 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java @@ -32,7 +32,7 @@ public interface RelationalJdbcConfiguration { /** * Explicitly configured database type. If not specified, the database type will be inferred from - * the JDBC connection metadata. Supported values: "postgresql", "cockroachdb", "h2" + * the JDBC connection metadata. Supported values: "postgresql", "cockroachdb", "h2", "mysql" */ Optional databaseType(); } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/idempotency/RelationalJdbcIdempotencyStore.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/idempotency/RelationalJdbcIdempotencyStore.java index 11be0b38136..f2c5f6918e6 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/idempotency/RelationalJdbcIdempotencyStore.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/idempotency/RelationalJdbcIdempotencyStore.java @@ -40,11 +40,13 @@ public class RelationalJdbcIdempotencyStore implements IdempotencyStore { private final DatasourceOperations datasourceOperations; + private final QueryGenerator queryGenerator; public RelationalJdbcIdempotencyStore( @NonNull DataSource dataSource, @NonNull RelationalJdbcConfiguration cfg) throws SQLException { this.datasourceOperations = new DatasourceOperations(dataSource, cfg); + this.queryGenerator = new QueryGenerator(datasourceOperations.getDatabaseType()); } @Override @@ -94,7 +96,7 @@ public ReserveResult reserve( public Optional load(String realmId, String idempotencyKey) { try { QueryGenerator.PreparedQuery query = - QueryGenerator.generateSelectQuery( + queryGenerator.generateSelectQuery( ModelIdempotencyRecord.ALL_COLUMNS, ModelIdempotencyRecord.TABLE_NAME, Map.of( @@ -150,7 +152,7 @@ public HeartbeatResult updateHeartbeat( } QueryGenerator.PreparedQuery update = - QueryGenerator.generateUpdateQuery( + queryGenerator.generateUpdateQuery( ModelIdempotencyRecord.ALL_COLUMNS, ModelIdempotencyRecord.TABLE_NAME, Map.of( @@ -213,7 +215,7 @@ public boolean finalizeRecord( whereEquals.put(ModelIdempotencyRecord.IDEMPOTENCY_KEY, idempotencyKey); QueryGenerator.PreparedQuery update = - QueryGenerator.generateUpdateQuery( + queryGenerator.generateUpdateQuery( ModelIdempotencyRecord.ALL_COLUMNS, ModelIdempotencyRecord.TABLE_NAME, setClause, @@ -234,7 +236,7 @@ public boolean finalizeRecord( public int purgeExpired(String realmId, Instant before) { try { QueryGenerator.PreparedQuery delete = - QueryGenerator.generateDeleteQuery( + queryGenerator.generateDeleteQuery( ModelIdempotencyRecord.ALL_COLUMNS, ModelIdempotencyRecord.TABLE_NAME, Map.of(ModelIdempotencyRecord.REALM_ID, realmId), diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/Converter.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/Converter.java index fcf5d1dfccd..4c6a3589dd0 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/Converter.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/Converter.java @@ -50,4 +50,11 @@ default PGobject toJsonbPGobject(String props) { throw new RuntimeException(e); } } + + default Object wrapJsonForDatabase(String json, DatabaseType databaseType) { + return switch (databaseType) { + case POSTGRES -> toJsonbPGobject(json); + case MYSQL, H2, COCKROACHDB -> json; + }; + } } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java index 10dc193869e..806ee59adb4 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelCommitMetricsReport.java @@ -23,6 +23,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.polaris.core.persistence.metrics.CommitMetricsRecord; import org.apache.polaris.immutables.PolarisImmutable; import org.apache.polaris.persistence.relational.jdbc.DatabaseType; @@ -104,6 +105,8 @@ public interface ModelCommitMetricsReport extends Converter JSON_COLUMNS = Set.of(METADATA); + // Getters String getReportId(); @@ -241,11 +244,8 @@ default Map toMap(DatabaseType databaseType) { map.put(TOTAL_DURATION_MS, getTotalDurationMs()); map.put(ATTEMPTS, getAttempts()); - if (databaseType.equals(DatabaseType.POSTGRES)) { - map.put(METADATA, toJsonbPGobject(getMetadata() != null ? getMetadata() : "{}")); - } else { - map.put(METADATA, getMetadata() != null ? getMetadata() : "{}"); - } + map.put( + METADATA, wrapJsonForDatabase(getMetadata() != null ? getMetadata() : "{}", databaseType)); return map; } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java index ef39529174c..e385ef72829 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java @@ -23,6 +23,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.polaris.core.entity.PolarisBaseEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; import org.apache.polaris.core.entity.PolarisEntitySubType; @@ -33,6 +34,8 @@ public class ModelEntity implements Converter { public static final String TABLE_NAME = "ENTITIES"; + public static final Set JSON_COLUMNS = Set.of("properties", "internal_properties"); + public static final String ID_COLUMN = "id"; private static final List ALL_COLUMNS = @@ -256,13 +259,8 @@ public Map toMap(DatabaseType databaseType) { map.put("purge_timestamp", this.getPurgeTimestamp()); map.put("to_purge_timestamp", this.getToPurgeTimestamp()); map.put("last_update_timestamp", this.getLastUpdateTimestamp()); - if (databaseType.equals(DatabaseType.POSTGRES)) { - map.put("properties", toJsonbPGobject(this.getProperties())); - map.put("internal_properties", toJsonbPGobject(this.getInternalProperties())); - } else { - map.put("properties", this.getProperties()); - map.put("internal_properties", this.getInternalProperties()); - } + map.put("properties", wrapJsonForDatabase(this.getProperties(), databaseType)); + map.put("internal_properties", wrapJsonForDatabase(this.getInternalProperties(), databaseType)); map.put("grant_records_version", this.getGrantRecordsVersion()); if (this.getSchemaVersion() >= 2) { map.put("location_without_scheme", this.getLocationWithoutScheme()); diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java index ff5f1111c5a..12329a5a4e9 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java @@ -24,6 +24,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.polaris.core.entity.EventEntity; import org.apache.polaris.immutables.PolarisImmutable; import org.apache.polaris.persistence.relational.jdbc.DatabaseType; @@ -55,6 +56,8 @@ public interface ModelEvent extends Converter { RESOURCE_IDENTIFIER, ADDITIONAL_PROPERTIES); + Set JSON_COLUMNS = Set.of(ADDITIONAL_PROPERTIES); + /** * Dummy instance to be used as a Converter when calling #fromResultSet(). * @@ -128,11 +131,7 @@ default Map toMap(DatabaseType databaseType) { map.put(PRINCIPAL_NAME, getPrincipalName()); map.put(RESOURCE_TYPE, getResourceType().toString()); map.put(RESOURCE_IDENTIFIER, getResourceIdentifier()); - if (databaseType.equals(DatabaseType.POSTGRES)) { - map.put(ADDITIONAL_PROPERTIES, toJsonbPGobject(getAdditionalProperties())); - } else { - map.put(ADDITIONAL_PROPERTIES, getAdditionalProperties()); - } + map.put(ADDITIONAL_PROPERTIES, wrapJsonForDatabase(getAdditionalProperties(), databaseType)); return map; } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPolicyMappingRecord.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPolicyMappingRecord.java index fdf3c3f0708..d285c5e2a3e 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPolicyMappingRecord.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPolicyMappingRecord.java @@ -23,6 +23,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.polaris.core.policy.PolarisPolicyMappingRecord; import org.apache.polaris.persistence.relational.jdbc.DatabaseType; @@ -38,6 +39,8 @@ public class ModelPolicyMappingRecord implements Converter JSON_COLUMNS = Set.of("parameters"); + // id of the catalog where target entity resides private long targetCatalogId; @@ -175,11 +178,7 @@ public Map toMap(DatabaseType databaseType) { map.put("policy_type_code", policyTypeCode); map.put("policy_catalog_id", policyCatalogId); map.put("policy_id", policyId); - if (databaseType.equals(DatabaseType.POSTGRES)) { - map.put("parameters", toJsonbPGobject(this.getParameters())); - } else { - map.put("parameters", this.getParameters()); - } + map.put("parameters", wrapJsonForDatabase(this.getParameters(), databaseType)); return map; } } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelRegistry.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelRegistry.java new file mode 100644 index 00000000000..896eaae8424 --- /dev/null +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelRegistry.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import java.util.Map; +import java.util.Set; + +/** + * Aggregates schema-level metadata across the {@code Model*} classes. + * + *

Currently the registry exposes the set of JSON-typed columns per table. JSON columns require + * database-specific WHERE-clause placeholder handling (MySQL needs {@code CAST(? AS JSON)} for + * structural equality), and {@code QueryGenerator} consults this registry instead of inspecting + * parameter values at runtime. + */ +public final class ModelRegistry { + + private static final Map> JSON_COLUMNS_BY_TABLE = + Map.of( + ModelEntity.TABLE_NAME, ModelEntity.JSON_COLUMNS, + ModelEvent.TABLE_NAME, ModelEvent.JSON_COLUMNS, + ModelPolicyMappingRecord.TABLE_NAME, ModelPolicyMappingRecord.JSON_COLUMNS, + ModelCommitMetricsReport.TABLE_NAME, ModelCommitMetricsReport.JSON_COLUMNS, + ModelScanMetricsReport.TABLE_NAME, ModelScanMetricsReport.JSON_COLUMNS); + + private ModelRegistry() {} + + /** + * Returns {@code true} when {@code columnName} is declared as a JSON column on the table named + * {@code tableName}. Tables that have no JSON columns (or are not registered) return {@code + * false}. + */ + public static boolean isJsonColumn(String tableName, String columnName) { + Set jsonColumns = JSON_COLUMNS_BY_TABLE.get(tableName); + return jsonColumns != null && jsonColumns.contains(columnName); + } +} diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java index 49ad75c6a86..8bedf00fcdb 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelScanMetricsReport.java @@ -23,6 +23,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import org.apache.polaris.core.persistence.metrics.ScanMetricsRecord; import org.apache.polaris.immutables.PolarisImmutable; @@ -104,6 +105,8 @@ public interface ModelScanMetricsReport extends Converter JSON_COLUMNS = Set.of(METADATA); + // Getters String getReportId(); @@ -241,11 +244,8 @@ default Map toMap(DatabaseType databaseType) { map.put(INDEXED_DELETE_FILES, getIndexedDeleteFiles()); map.put(TOTAL_DELETE_FILE_SIZE_BYTES, getTotalDeleteFileSizeBytes()); - if (databaseType.equals(DatabaseType.POSTGRES)) { - map.put(METADATA, toJsonbPGobject(getMetadata() != null ? getMetadata() : "{}")); - } else { - map.put(METADATA, getMetadata() != null ? getMetadata() : "{}"); - } + map.put( + METADATA, wrapJsonForDatabase(getMetadata() != null ? getMetadata() : "{}", databaseType)); return map; } diff --git a/persistence/relational-jdbc/src/main/resources/mysql/schema-v4.sql b/persistence/relational-jdbc/src/main/resources/mysql/schema-v4.sql new file mode 100644 index 00000000000..20c75e60c96 --- /dev/null +++ b/persistence/relational-jdbc/src/main/resources/mysql/schema-v4.sql @@ -0,0 +1,254 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file-- +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"). You may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- MySQL schema v4 (matching PostgreSQL schema v4) +-- Design notes: +-- * The POLARIS_SCHEMA database must be pre-created and the user must have privileges on it. +-- * In MySQL, schema is synonymous with database. +-- * Indexes are declared inside CREATE TABLE because MySQL does not support +-- "CREATE INDEX IF NOT EXISTS" — the bootstrap runs this script once per realm +-- and "CREATE TABLE IF NOT EXISTS" handles the idempotency. +-- * Uses VARCHAR(255) for PRIMARY KEY / UNIQUE KEY columns where PostgreSQL uses TEXT +-- (MySQL forbids TEXT columns in keys without a prefix length). +-- * Uses JSON type instead of JSONB. +-- * Uses ON DUPLICATE KEY UPDATE instead of ON CONFLICT ... DO UPDATE. +-- * Table identifiers use the same case as the Java `Model*.TABLE_NAME` constants +-- (UPPERCASE for everything except `idempotency_records`, which is lowercase to +-- match `ModelIdempotencyRecord.TABLE_NAME`). This keeps the schema working on +-- default Linux MySQL (`lower_case_table_names=0`, case-sensitive identifiers) +-- without any server-level flag. +-- * Every table sets `DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin` so string +-- comparisons are case-sensitive/binary, matching the PostgreSQL `TEXT` semantics +-- expected by Polaris (e.g. `entities.name` must treat `foo` and `Foo` as distinct). + +USE POLARIS_SCHEMA; + +CREATE TABLE IF NOT EXISTS VERSION ( + version_key VARCHAR(255) PRIMARY KEY COMMENT 'version key', + version_value INTEGER NOT NULL COMMENT 'version value' +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin COMMENT = 'the version of the JDBC schema in use'; + +INSERT INTO VERSION (version_key, version_value) +VALUES ('version', 4) +ON DUPLICATE KEY UPDATE version_value = VALUES(version_value); + +CREATE TABLE IF NOT EXISTS ENTITIES ( + realm_id VARCHAR(255) NOT NULL COMMENT 'realm_id used for multi-tenancy', + catalog_id BIGINT NOT NULL COMMENT 'catalog id', + id BIGINT NOT NULL COMMENT 'entity id', + parent_id BIGINT NOT NULL COMMENT 'entity id of parent', + -- Matches Polaris's MAX_IDENTIFIER_LENGTH = 256. Keeps the UNIQUE + -- (realm_id, ..., name) index well within the InnoDB 3072-byte key-length + -- limit: 255*4 + 8+8+4 + 256*4 = 2064 bytes (utf8mb4). + name VARCHAR(256) NOT NULL COMMENT 'entity name', + entity_version INT NOT NULL COMMENT 'version of the entity', + type_code INT NOT NULL COMMENT 'type code', + sub_type_code INT NOT NULL COMMENT 'sub type of entity', + create_timestamp BIGINT NOT NULL COMMENT 'creation time of entity', + drop_timestamp BIGINT NOT NULL COMMENT 'time of drop of entity', + purge_timestamp BIGINT NOT NULL COMMENT 'time to start purging entity', + to_purge_timestamp BIGINT NOT NULL, + last_update_timestamp BIGINT NOT NULL COMMENT 'last time the entity is touched', + properties JSON NOT NULL DEFAULT ('{}') COMMENT 'entities properties json', + internal_properties JSON NOT NULL DEFAULT ('{}') COMMENT 'entities internal properties json', + grant_records_version INT NOT NULL COMMENT 'the version of grant records change on the entity', + location_without_scheme TEXT, + PRIMARY KEY (realm_id, id), + CONSTRAINT constraint_name UNIQUE (realm_id, catalog_id, parent_id, type_code, name), + INDEX idx_entities (realm_id, catalog_id, id), + INDEX idx_entities_catalog_id_id (catalog_id, id), + -- location_without_scheme is TEXT, use prefix length 400 to stay under + -- the InnoDB 3072-byte key length limit (utf8mb4 = 4 bytes/char). + INDEX idx_locations (realm_id, parent_id, location_without_scheme(400)) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin COMMENT = 'all the entities'; + +CREATE TABLE IF NOT EXISTS GRANT_RECORDS ( + realm_id VARCHAR(255) NOT NULL, + securable_catalog_id BIGINT NOT NULL COMMENT 'catalog id of the securable', + securable_id BIGINT NOT NULL COMMENT 'entity id of the securable', + grantee_catalog_id BIGINT NOT NULL COMMENT 'catalog id of the grantee', + grantee_id BIGINT NOT NULL COMMENT 'id of the grantee', + privilege_code INTEGER COMMENT 'privilege code', + PRIMARY KEY (realm_id, securable_catalog_id, securable_id, grantee_catalog_id, grantee_id, privilege_code), + INDEX idx_grants_realm_grantee (realm_id, grantee_id), + INDEX idx_grants_realm_securable (realm_id, securable_id) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin COMMENT = 'grant records for entities'; + +CREATE TABLE IF NOT EXISTS PRINCIPAL_AUTHENTICATION_DATA ( + realm_id VARCHAR(255) NOT NULL, + principal_id BIGINT NOT NULL, + principal_client_id VARCHAR(255) NOT NULL, + main_secret_hash VARCHAR(255) NOT NULL, + secondary_secret_hash VARCHAR(255) NOT NULL, + secret_salt VARCHAR(255) NOT NULL, + PRIMARY KEY (realm_id, principal_client_id) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin COMMENT = 'authentication data for client'; + +CREATE TABLE IF NOT EXISTS POLICY_MAPPING_RECORD ( + realm_id VARCHAR(255) NOT NULL, + target_catalog_id BIGINT NOT NULL, + target_id BIGINT NOT NULL, + policy_type_code INTEGER NOT NULL, + policy_catalog_id BIGINT NOT NULL, + policy_id BIGINT NOT NULL, + parameters JSON NOT NULL DEFAULT ('{}'), + PRIMARY KEY (realm_id, target_catalog_id, target_id, policy_type_code, policy_catalog_id, policy_id), + INDEX idx_policy_mapping_record (realm_id, policy_type_code, policy_catalog_id, policy_id, target_catalog_id, target_id) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; + +CREATE TABLE IF NOT EXISTS EVENTS ( + realm_id VARCHAR(255) NOT NULL, + catalog_id VARCHAR(255) NOT NULL, + event_id VARCHAR(255) NOT NULL, + request_id VARCHAR(255), + event_type VARCHAR(255) NOT NULL, + timestamp_ms BIGINT NOT NULL, + principal_name VARCHAR(255), + resource_type VARCHAR(255) NOT NULL, + resource_identifier VARCHAR(1024) NOT NULL, + additional_properties JSON NOT NULL DEFAULT ('{}'), + PRIMARY KEY (event_id) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; + +-- Idempotency records (key-only idempotency; durable replay). +-- This is the one table whose Java constant (`ModelIdempotencyRecord.TABLE_NAME`) is +-- lowercase; matching the Java case is what lets the MySQL backend run on default +-- Linux MySQL without `lower_case_table_names=1`. +CREATE TABLE IF NOT EXISTS idempotency_records ( + realm_id VARCHAR(255) NOT NULL, + idempotency_key VARCHAR(255) NOT NULL, + operation_type VARCHAR(255) NOT NULL, + resource_id VARCHAR(1024) NOT NULL, + + http_status INTEGER, + error_subtype VARCHAR(255), + response_summary TEXT, + response_headers TEXT, + finalized_at TIMESTAMP NULL, + + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + heartbeat_at TIMESTAMP NULL, + executor_id VARCHAR(255), + expires_at TIMESTAMP NULL, + + PRIMARY KEY (realm_id, idempotency_key), + INDEX idx_idemp_realm_expires (realm_id, expires_at) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; + +-- ============================================================================ +-- SCAN METRICS REPORT TABLE +-- ============================================================================ + +CREATE TABLE IF NOT EXISTS SCAN_METRICS_REPORT ( + report_id VARCHAR(255) NOT NULL, + realm_id VARCHAR(255) NOT NULL, + catalog_id BIGINT NOT NULL, + table_id BIGINT NOT NULL, + + timestamp_ms BIGINT NOT NULL, + principal_name VARCHAR(255), + request_id VARCHAR(255), + + otel_trace_id VARCHAR(255), + otel_span_id VARCHAR(255), + report_trace_id VARCHAR(255), + + snapshot_id BIGINT, + schema_id INTEGER, + filter_expression TEXT, + projected_field_ids TEXT, + projected_field_names TEXT, + + result_data_files BIGINT DEFAULT 0, + result_delete_files BIGINT DEFAULT 0, + total_file_size_bytes BIGINT DEFAULT 0, + total_data_manifests BIGINT DEFAULT 0, + total_delete_manifests BIGINT DEFAULT 0, + scanned_data_manifests BIGINT DEFAULT 0, + scanned_delete_manifests BIGINT DEFAULT 0, + skipped_data_manifests BIGINT DEFAULT 0, + skipped_delete_manifests BIGINT DEFAULT 0, + skipped_data_files BIGINT DEFAULT 0, + skipped_delete_files BIGINT DEFAULT 0, + total_planning_duration_ms BIGINT DEFAULT 0, + + equality_delete_files BIGINT DEFAULT 0, + positional_delete_files BIGINT DEFAULT 0, + indexed_delete_files BIGINT DEFAULT 0, + total_delete_file_size_bytes BIGINT DEFAULT 0, + + metadata JSON DEFAULT ('{}'), + + PRIMARY KEY (realm_id, report_id), + INDEX idx_scan_report_timestamp (realm_id, timestamp_ms), + INDEX idx_scan_report_lookup (realm_id, catalog_id, table_id, timestamp_ms) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin COMMENT = 'Scan metrics reports as first-class entities'; + +-- ============================================================================ +-- COMMIT METRICS REPORT TABLE +-- ============================================================================ + +CREATE TABLE IF NOT EXISTS COMMIT_METRICS_REPORT ( + report_id VARCHAR(255) NOT NULL, + realm_id VARCHAR(255) NOT NULL, + catalog_id BIGINT NOT NULL, + table_id BIGINT NOT NULL, + + timestamp_ms BIGINT NOT NULL, + principal_name VARCHAR(255), + request_id VARCHAR(255), + + otel_trace_id VARCHAR(255), + otel_span_id VARCHAR(255), + report_trace_id VARCHAR(255), + + snapshot_id BIGINT NOT NULL, + sequence_number BIGINT, + operation VARCHAR(255) NOT NULL, + + added_data_files BIGINT DEFAULT 0, + removed_data_files BIGINT DEFAULT 0, + total_data_files BIGINT DEFAULT 0, + added_delete_files BIGINT DEFAULT 0, + removed_delete_files BIGINT DEFAULT 0, + total_delete_files BIGINT DEFAULT 0, + + added_equality_delete_files BIGINT DEFAULT 0, + removed_equality_delete_files BIGINT DEFAULT 0, + + added_positional_delete_files BIGINT DEFAULT 0, + removed_positional_delete_files BIGINT DEFAULT 0, + + added_records BIGINT DEFAULT 0, + removed_records BIGINT DEFAULT 0, + total_records BIGINT DEFAULT 0, + + added_file_size_bytes BIGINT DEFAULT 0, + removed_file_size_bytes BIGINT DEFAULT 0, + total_file_size_bytes BIGINT DEFAULT 0, + + total_duration_ms BIGINT DEFAULT 0, + attempts INTEGER DEFAULT 1, + + metadata JSON DEFAULT ('{}'), + + PRIMARY KEY (realm_id, report_id), + INDEX idx_commit_report_timestamp (realm_id, timestamp_ms), + INDEX idx_commit_report_lookup (realm_id, catalog_id, table_id, timestamp_ms) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin COMMENT = 'Commit metrics reports as first-class entities'; diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java index 38a06f2b5fa..095d0aed04a 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/QueryGeneratorTest.java @@ -41,6 +41,10 @@ public class QueryGeneratorTest { private static final String REALM_ID = "testRealm"; + // Bound to H2 for the bulk of the assertions; tests that need a different DatabaseType + // construct their own instance. + private final QueryGenerator queryGenerator = new QueryGenerator(DatabaseType.H2); + @Test void testGenerateSelectQuery_withMaQueryGeneratorpWhereClause() { Map whereClause = new HashMap<>(); @@ -50,7 +54,8 @@ void testGenerateSelectQuery_withMaQueryGeneratorpWhereClause() { "SELECT id, catalog_id, parent_id, type_code, name, entity_version, sub_type_code, create_timestamp, drop_timestamp, purge_timestamp, to_purge_timestamp, last_update_timestamp, properties, internal_properties, grant_records_version, location_without_scheme FROM POLARIS_SCHEMA.ENTITIES WHERE entity_version = ? AND name = ?"; assertEquals( expectedQuery, - QueryGenerator.generateSelectQuery( + queryGenerator + .generateSelectQuery( ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, whereClause) .sql()); } @@ -123,7 +128,8 @@ void testGenerateUpdateQuery_nonNullFields() { "UPDATE POLARIS_SCHEMA.ENTITIES SET id = ?, catalog_id = ?, parent_id = ?, type_code = ?, name = ?, entity_version = ?, sub_type_code = ?, create_timestamp = ?, drop_timestamp = ?, purge_timestamp = ?, to_purge_timestamp = ?, last_update_timestamp = ?, properties = ?, internal_properties = ?, grant_records_version = ?, location_without_scheme = ? WHERE id = ?"; assertEquals( expectedQuery, - QueryGenerator.generateUpdateQuery( + queryGenerator + .generateUpdateQuery( ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, entity.toMap(DatabaseType.H2).values().stream().toList(), @@ -140,7 +146,8 @@ void testGenerateUpdateQuery_partialNonNullFields() { "UPDATE POLARIS_SCHEMA.ENTITIES SET id = ?, catalog_id = ?, parent_id = ?, type_code = ?, name = ?, entity_version = ?, sub_type_code = ?, create_timestamp = ?, drop_timestamp = ?, purge_timestamp = ?, to_purge_timestamp = ?, last_update_timestamp = ?, properties = ?, internal_properties = ?, grant_records_version = ?, location_without_scheme = ? WHERE id = ?"; assertEquals( expectedQuery, - QueryGenerator.generateUpdateQuery( + queryGenerator + .generateUpdateQuery( ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, entity.toMap(DatabaseType.H2).values().stream().toList(), @@ -155,7 +162,8 @@ void testGenerateDeleteQuery_withMapWhereClause() { String expectedQuery = "DELETE FROM POLARIS_SCHEMA.ENTITIES WHERE name = ?"; assertEquals( expectedQuery, - QueryGenerator.generateDeleteQuery( + queryGenerator + .generateDeleteQuery( ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, whereClause) .sql()); } @@ -165,7 +173,8 @@ void testGenerateDeleteQuery_withStringWhereClause() { String expectedQuery = "DELETE FROM POLARIS_SCHEMA.ENTITIES WHERE name = ?"; assertEquals( expectedQuery, - QueryGenerator.generateDeleteQuery( + queryGenerator + .generateDeleteQuery( ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, Map.of("name", "oldName")) .sql()); } @@ -180,8 +189,8 @@ void testGenerateDeleteQuery_byObject() { "DELETE FROM POLARIS_SCHEMA.ENTITIES WHERE id = ? AND catalog_id = ? AND parent_id = ? AND type_code = ? AND name = ? AND entity_version = ? AND sub_type_code = ? AND create_timestamp = ? AND drop_timestamp = ? AND purge_timestamp = ? AND to_purge_timestamp = ? AND last_update_timestamp = ? AND properties = ? AND internal_properties = ? AND grant_records_version = ? AND location_without_scheme = ? AND realm_id = ?"; assertEquals( expectedQuery, - QueryGenerator.generateDeleteQuery( - ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, objMap) + queryGenerator + .generateDeleteQuery(ModelEntity.getAllColumnNames(2), ModelEntity.TABLE_NAME, objMap) .sql()); } @@ -191,7 +200,9 @@ void testGenerateWhereClause_singleCondition() { whereClause.put("name", "test"); assertEquals( " WHERE name = ?", - QueryGenerator.generateWhereClause(Set.of("name"), whereClause, Map.of()).sql()); + queryGenerator + .generateWhereClause(ModelEntity.TABLE_NAME, Set.of("name"), whereClause, Map.of()) + .sql()); } @Test @@ -201,7 +212,10 @@ void testGenerateWhereClause_multipleConditions() { whereClause.put("version", 1); assertEquals( " WHERE name = ? AND version = ?", - QueryGenerator.generateWhereClause(Set.of("name", "version"), whereClause, Map.of()).sql()); + queryGenerator + .generateWhereClause( + ModelEntity.TABLE_NAME, Set.of("name", "version"), whereClause, Map.of()) + .sql()); } @Test @@ -211,15 +225,23 @@ void testGenerateWhereClause_multipleConditions_AndInequality() { whereClause.put("version", 1); assertEquals( " WHERE name = ? AND version = ? AND id > ?", - QueryGenerator.generateWhereClause( - Set.of("name", "version", "id"), whereClause, Map.of("id", 123)) + queryGenerator + .generateWhereClause( + ModelEntity.TABLE_NAME, + Set.of("name", "version", "id"), + whereClause, + Map.of("id", 123)) .sql()); } @Test void testGenerateWhereClause_emptyMap() { Map whereClause = Collections.emptyMap(); - assertEquals("", QueryGenerator.generateWhereClause(Set.of(), whereClause, Map.of()).sql()); + assertEquals( + "", + queryGenerator + .generateWhereClause(ModelEntity.TABLE_NAME, Set.of(), whereClause, Map.of()) + .sql()); } @Test @@ -235,7 +257,8 @@ void testGenerateWhereClauseExtended_allPredicatesAndStableParameterOrder() { Set whereIsNotNull = new LinkedHashSet<>(List.of("e")); QueryGenerator.QueryFragment where = - QueryGenerator.generateWhereClauseExtended( + queryGenerator.generateWhereClauseExtended( + "test_table", Set.of("a", "b", "c", "d", "e"), whereEquals, whereGreater, @@ -247,6 +270,26 @@ void testGenerateWhereClauseExtended_allPredicatesAndStableParameterOrder() { Assertions.assertThat(where.parameters()).containsExactly("A", 2, 3); } + @Test + void testGenerateWhereClause_mysqlJsonColumn_emitsCastPlaceholder() { + // POLICY_MAPPING_RECORD.parameters is declared as JSON in ModelRegistry; MySQL needs + // CAST(? AS JSON) for structural equality, other backends keep the plain ? placeholder. + Map whereEquals = new LinkedHashMap<>(); + whereEquals.put("parameters", "{\"a\":1}"); + + QueryGenerator mysqlGenerator = new QueryGenerator(DatabaseType.MYSQL); + QueryGenerator.QueryFragment mysqlFragment = + mysqlGenerator.generateWhereClause( + "POLICY_MAPPING_RECORD", Set.of("parameters"), whereEquals, Map.of()); + assertEquals(" WHERE parameters = CAST(? AS JSON)", mysqlFragment.sql()); + + QueryGenerator postgresGenerator = new QueryGenerator(DatabaseType.POSTGRES); + QueryGenerator.QueryFragment postgresFragment = + postgresGenerator.generateWhereClause( + "POLICY_MAPPING_RECORD", Set.of("parameters"), whereEquals, Map.of()); + assertEquals(" WHERE parameters = ?", postgresFragment.sql()); + } + @Test void testGenerateUpdateQueryExtended_supportsNullSetValues() { Map setClause = new LinkedHashMap<>(); @@ -261,7 +304,7 @@ void testGenerateUpdateQueryExtended_supportsNullSetValues() { whereLess.put("http_status", 500); QueryGenerator.PreparedQuery q = - QueryGenerator.generateUpdateQuery( + queryGenerator.generateUpdateQuery( List.of("error_subtype", "http_status", "realm_id", "idempotency_key", "executor_id"), "idempotency_records", setClause, @@ -283,7 +326,7 @@ void testGenerateUpdateQueryExtended_rejectsEmptySetClause() { assertThrows( IllegalArgumentException.class, () -> - QueryGenerator.generateUpdateQuery( + queryGenerator.generateUpdateQuery( List.of("a"), "t", Map.of(), @@ -297,7 +340,7 @@ void testGenerateUpdateQueryExtended_rejectsEmptySetClause() { @Test void testGenerateDeleteQueryExtended_includesNullPredicatesAndLessThan() { QueryGenerator.PreparedQuery q = - QueryGenerator.generateDeleteQuery( + queryGenerator.generateDeleteQuery( List.of("realm_id", "expires_at", "finalized_at"), "idempotency_records", Map.of("realm_id", "r1"), @@ -315,7 +358,7 @@ void testGenerateDeleteQueryExtended_includesNullPredicatesAndLessThan() { @Test void testGenerateDeleteQueryExtended_allowsRealmIdEvenIfNotInTableColumns() { QueryGenerator.PreparedQuery q = - QueryGenerator.generateDeleteQuery( + queryGenerator.generateDeleteQuery( List.of("id"), "some_table", Map.of("realm_id", "r1"), @@ -336,7 +379,7 @@ void testGenerateUpdateQueryExtended_rejectsInvalidColumns() { assertThrows( IllegalArgumentException.class, () -> - QueryGenerator.generateUpdateQuery( + queryGenerator.generateUpdateQuery( List.of("a"), "t", setClause, diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SchemaVersions.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SchemaVersions.java index ddebc385e6a..8893abf5659 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SchemaVersions.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SchemaVersions.java @@ -77,6 +77,7 @@ private static int firstSchemaVersion(DatabaseType databaseType) { return switch (databaseType) { case H2 -> 0; case POSTGRES, COCKROACHDB -> 1; + case MYSQL -> 4; // MySQL ships schema-v4.sql only }; } } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SimpleRelationalJdbcConfiguration.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SimpleRelationalJdbcConfiguration.java index bbc28c12d81..6edd64283ae 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SimpleRelationalJdbcConfiguration.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SimpleRelationalJdbcConfiguration.java @@ -34,6 +34,7 @@ static SimpleRelationalJdbcConfiguration forDatabaseType(DatabaseType databaseTy case H2 -> new SimpleRelationalJdbcConfiguration("h2"); case POSTGRES -> new SimpleRelationalJdbcConfiguration("postgresql"); case COCKROACHDB -> new SimpleRelationalJdbcConfiguration("cockroachdb"); + case MYSQL -> new SimpleRelationalJdbcConfiguration("mysql"); }; } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelRegistryTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelRegistryTest.java new file mode 100644 index 00000000000..8c1fbe7e781 --- /dev/null +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/models/ModelRegistryTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.persistence.relational.jdbc.models; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class ModelRegistryTest { + + @Test + void entityJsonColumnsAreRegistered() { + assertThat(ModelRegistry.isJsonColumn(ModelEntity.TABLE_NAME, "properties")).isTrue(); + assertThat(ModelRegistry.isJsonColumn(ModelEntity.TABLE_NAME, "internal_properties")).isTrue(); + } + + @Test + void eventJsonColumnsAreRegistered() { + assertThat(ModelRegistry.isJsonColumn(ModelEvent.TABLE_NAME, "additional_properties")).isTrue(); + } + + @Test + void policyMappingRecordJsonColumnsAreRegistered() { + assertThat(ModelRegistry.isJsonColumn(ModelPolicyMappingRecord.TABLE_NAME, "parameters")) + .isTrue(); + } + + @Test + void metricsReportJsonColumnsAreRegistered() { + assertThat(ModelRegistry.isJsonColumn(ModelCommitMetricsReport.TABLE_NAME, "metadata")) + .isTrue(); + assertThat(ModelRegistry.isJsonColumn(ModelScanMetricsReport.TABLE_NAME, "metadata")).isTrue(); + } + + @Test + void nonJsonColumnsReturnFalse() { + assertThat(ModelRegistry.isJsonColumn(ModelEntity.TABLE_NAME, "id")).isFalse(); + assertThat(ModelRegistry.isJsonColumn(ModelEntity.TABLE_NAME, "name")).isFalse(); + assertThat(ModelRegistry.isJsonColumn(ModelPolicyMappingRecord.TABLE_NAME, "policy_id")) + .isFalse(); + } + + @Test + void unregisteredTableReturnsFalse() { + assertThat(ModelRegistry.isJsonColumn("UNKNOWN_TABLE", "any_column")).isFalse(); + } +} diff --git a/runtime/defaults/src/main/resources/application.properties b/runtime/defaults/src/main/resources/application.properties index d7b9196c380..73e5a99cc50 100644 --- a/runtime/defaults/src/main/resources/application.properties +++ b/runtime/defaults/src/main/resources/application.properties @@ -51,6 +51,16 @@ quarkus.otel.enabled=true #quarkus.mongodb.connection-string=mongodb://localhost:27017 quarkus.datasource.db-kind=postgresql +# Named datasource for MySQL (the one relational backend that does not share the PostgreSQL +# JDBC driver). `jdbc=false` skips driver-extension validation when the GPL-licensed MySQL +# driver is absent from the build; `-PincludeMysqlDriver=true` flips it back to `true`. +# `active=false` keeps the bean inactive by default; users opt in by setting +# `polaris.persistence.relational.jdbc.database-type=mysql` together with +# `quarkus.datasource.mysql.active=true` and the connection properties. +quarkus.datasource.mysql.db-kind=mysql +quarkus.datasource.mysql.jdbc=false +quarkus.datasource.mysql.active=false + # ---- Runtime Configuration ---- # Below are default values for properties that can be changed in runtime. diff --git a/runtime/server/README.md b/runtime/server/README.md index 640d2e1667b..127f0951ca4 100644 --- a/runtime/server/README.md +++ b/runtime/server/README.md @@ -64,3 +64,7 @@ following command: -Dquarkus.container-image.group=apache \ -Dquarkus.container-image.name=polaris-local ``` + +## MySQL support + +MySQL is available for the Relational JDBC backend via an opt-in, build-from-source path: the GPL-licensed MySQL JDBC driver (ASF Category X) is not bundled in the official Polaris artifacts. Because this is a custom downstream build rather than part of the standard server, the build, configuration and bootstrap details live in separate JDBC/MySQL documentation: see [`persistence/relational-jdbc/MYSQL.md`](../../persistence/relational-jdbc/MYSQL.md). diff --git a/runtime/server/build.gradle.kts b/runtime/server/build.gradle.kts index 943394567b6..a51453e1df0 100644 --- a/runtime/server/build.gradle.kts +++ b/runtime/server/build.gradle.kts @@ -59,6 +59,26 @@ dependencies { implementation("io.quarkus:quarkus-container-image-docker") } +// MySQL is opt-in: the GPL JDBC driver (ASF Category X) is not bundled in the default +// build. `runtime/service` does not bring the driver transitively, so the default +// release build stays GPL-free without any classpath exclusions. The driver is added +// directly here only when `-PincludeMysqlDriver=true` is provided. +// +// Verification contract for the opt-in `-PincludeMysqlDriver=true` build: +// - Supported: `:polaris-server:assemble`, Quarkus image build tasks. +// - NOT supported (will fail by design because the GPL MySQL JDBC driver is +// intentionally absent from `LICENSE`): +// `:polaris-server:build`, `:polaris-server:check`, +// `:polaris-server:publishToMavenLocal`, `:polaris-server:generateLicenseReport`, +// `:polaris-server:checkLicense`, `:polaris-server:licenseReportZip`. +// See `runtime/server/README.md` for the downstream/custom-build framing. +if (project.hasProperty("includeMysqlDriver")) { + dependencies { runtimeOnly("io.quarkus:quarkus-jdbc-mysql") } + // Override the `jdbc=false` default in `application.properties` so Quarkus wires up + // the MySQL named datasource at build time. + quarkus { quarkusBuildProperties.put("quarkus.datasource.mysql.jdbc", "true") } +} + quarkus { quarkusBuildProperties.put("quarkus.package.jar.type", "fast-jar") // Pull manifest attributes from the "main" `jar` task to get the diff --git a/site/content/in-dev/unreleased/metastores/relational-jdbc.md b/site/content/in-dev/unreleased/metastores/relational-jdbc.md index 037cb0fafed..6acabff2195 100644 --- a/site/content/in-dev/unreleased/metastores/relational-jdbc.md +++ b/site/content/in-dev/unreleased/metastores/relational-jdbc.md @@ -68,7 +68,13 @@ quarkus.rds.credentials-provider.aws.port=6160 This is the basic configuration. For more details, please refer to the [Quarkus plugin documentation](https://docs.quarkiverse.io/quarkus-amazon-services/dev/amazon-rds.html#_configuration_reference). -The Relational JDBC metastore currently relies on a Quarkus-managed datasource and supports only PostgresSQL and H2 databases. At this time, official documentation is provided exclusively for usage with PostgreSQL. +## 3. MySQL (custom source build) + +The relational JDBC backend can be built with MySQL 8.0+ support, but the MySQL JDBC driver is GPL-licensed and is not bundled in the official Polaris release artifacts (see [issue #2491](https://github.com/apache/polaris/issues/2491)). To use this path, download the official Polaris source release and build your own derivative from that source tree; see `persistence/relational-jdbc/MYSQL.md` in the unpacked source release for build and configuration details. The runner you build is a custom downstream derivative, not an official Polaris artifact. + +--- + +The Relational JDBC metastore currently relies on a Quarkus-managed datasource. Official Polaris release artifacts support PostgreSQL and H2; MySQL is available only via a custom source build as described above. At this time, the most detailed documentation is provided for PostgreSQL. Please refer to the documentation here: [Configure data sources in Quarkus](https://quarkus.io/guides/datasource).