Add MySQL as a supported database for the relational JDBC persistence backend

CHANGELOG.md

@@ -98,6 +98,7 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
 - Added support for multiple event listeners. Set `polaris.event-listener.types` to a comma-separated list of event listener types to enable multiple event listeners.
 - Added support for enabling only a subset of event types and event categories per event listener. Set `polaris.event-listener.`_`<name>`_`.enabled-event-types` or `polaris.event-listener.`_`<name>`_`.enabled-event-categories` to the list of event types or categories for the specified event listener to only consume the selected subset of events.
 - Added support for **Apache Ranger** as an external authorizer (Beta).
+- Added source-level support for building a MySQL-capable Polaris server for the relational JDBC backend. Official Polaris release artifacts do not include the MySQL JDBC driver, which is GPL-licensed. Users who need this path should download the official Polaris source release and build their own derivative from that source tree (`./gradlew :polaris-server:assemble -PincludeMysqlDriver=true`); see `runtime/server/README.md` in the unpacked source release for build details.
 
 ### Changes
 - Improved Python CLI error messages and exit codes for invalid arguments and configuration errors.

bom/build.gradle.kts

@@ -137,6 +137,9 @@ tasks.register<VerifyBomDependenciesTask>("verifyBomDependencies") {
       ":aggregated-license-report",
       ":polaris-config-docs-site",
       ":polaris-distribution",
+      // Integration-test-only module that pulls the GPL MySQL JDBC driver; it is not a
+      // published library, so it must not be referenced from the BOM.
+      ":polaris-relational-jdbc-mysql-tests",
     )
   )
 }

gradle/projects.main.properties

@@ -33,6 +33,7 @@ polaris-admin=runtime/admin
 polaris-runtime-common=runtime/common
 polaris-runtime-test-common=runtime/test-common
 polaris-relational-jdbc=persistence/relational-jdbc
+polaris-relational-jdbc-mysql-tests=persistence/relational-jdbc-mysql/tests
 polaris-tests=integration-tests
 aggregated-license-report=aggregated-license-report
 polaris-immutables=tools/immutables

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java

@@ -325,6 +325,55 @@ public void testCreateCatalogWithGcpStorageConfig() {
     }
   }
 
+  @Test
+  public void testCatalogsDifferingOnlyByCaseCoexist() {
+    // Cross-backend regression check that identifier uniqueness is case-sensitive on the
+    // `entities` table. Notably this catches MySQL collation regressions: a default
+    // case-insensitive collation would collapse `foo` and `Foo` into a single row via the
+    // (realm_id, catalog_id, parent_id, type_code, name) unique constraint, diverging from
+    // PostgreSQL's case-sensitive TEXT semantics.
+    AwsStorageConfigInfo awsConfigModel =
+        AwsStorageConfigInfo.builder()
+            .setRoleArn("arn:aws:iam::000000000000:role/polaris-it")
+            .setStorageType(StorageConfigInfo.StorageTypeEnum.S3)
+            .setAllowedLocations(List.of("s3://test-bucket/"))
+            .build();
+    String lowerName = client.newEntityName("foo");
+    String upperName = lowerName.replaceFirst("foo", "Foo");
+    Catalog lowerCatalog =
+        PolarisCatalog.builder()
+            .setType(Catalog.TypeEnum.INTERNAL)
+            .setName(lowerName)
+            .setProperties(new CatalogProperties("s3://test-bucket/" + lowerName))
+            .setStorageConfigInfo(awsConfigModel)
+            .build();
+    Catalog upperCatalog =
+        PolarisCatalog.builder()
+            .setType(Catalog.TypeEnum.INTERNAL)
+            .setName(upperName)
+            .setProperties(new CatalogProperties("s3://test-bucket/" + upperName))
+            .setStorageConfigInfo(awsConfigModel)
+            .build();
+    managementApi.createCatalog(lowerCatalog);
+    try {
+      managementApi.createCatalog(upperCatalog);
+      try {
+        try (Response r = managementApi.request("v1/catalogs/" + lowerName).get()) {
+          assertThat(r).returns(Response.Status.OK.getStatusCode(), Response::getStatus);
+          assertThat(r.readEntity(Catalog.class).getName()).isEqualTo(lowerName);
+        }
+        try (Response r = managementApi.request("v1/catalogs/" + upperName).get()) {
+          assertThat(r).returns(Response.Status.OK.getStatusCode(), Response::getStatus);
+          assertThat(r.readEntity(Catalog.class).getName()).isEqualTo(upperName);
+        }
+      } finally {
+        managementApi.deleteCatalog(upperName);
+      }
+    } finally {
+      managementApi.deleteCatalog(lowerName);
+    }
+  }
+
   @Test
   public void testCreateCatalogWithNullBaseLocation() {
     AwsStorageConfigInfo awsConfigModel =

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisPolicyServiceIntegrationTest.java

@@ -474,6 +474,47 @@ public void testDropPolicy() {
         .hasSize(0);
   }
 
+  @Test
+  public void testAttachAndDetachPolicyWithNonTrivialJsonParameters() {
+    // Regression coverage for `JdbcBasePersistenceImpl.deleteFromPolicyMappingRecords`:
+    // the DELETE WHERE clause passes the `parameters` JSON column value verbatim and
+    // must match the stored row. On MySQL this requires `CAST(? AS JSON)` placeholder
+    // selection (via `ModelRegistry.isJsonColumn` + `DatabaseType.asJsonConditionPlaceholder`);
+    // on PostgreSQL the `parameters` column is jsonb and matches with a plain `?`
+    // placeholder bound to a `PGobject(jsonb)` value. Either way, attaching with
+    // non-trivial JSON parameters and then detaching must succeed.
+    restCatalog.createNamespace(NS1);
+    policyApi.createPolicy(
+        currentCatalogName,
+        NS1_P1,
+        PredefinedPolicyTypes.DATA_COMPACTION,
+        EXAMPLE_TABLE_MAINTENANCE_POLICY_CONTENT,
+        "test policy");
+
+    PolicyAttachmentTarget catalogTarget =
+        PolicyAttachmentTarget.builder().setType(PolicyAttachmentTarget.TypeEnum.CATALOG).build();
+    Map<String, String> nonTrivialParameters =
+        Map.of("retention", "30days", "scope", "namespace-and-tables", "owner", "polaris-it");
+    policyApi.attachPolicy(currentCatalogName, NS1_P1, catalogTarget, nonTrivialParameters);
+
+    // Detach exercises `deleteFromPolicyMappingRecords` which uses the `parameters`
+    // JSON column in the DELETE WHERE clause. A regression that drops the JSON
+    // placeholder selection (or that re-introduces `Converter.MysqlJsonValue`-based
+    // dispatch in the wrong way) would silently fail to delete the row on MySQL.
+    policyApi.detachPolicy(currentCatalogName, NS1_P1, catalogTarget);
+
+    // Re-attaching the same target with different non-trivial parameters must succeed
+    // (the previous row must have actually been deleted, not just orphaned).
+    policyApi.attachPolicy(
+        currentCatalogName,
+        NS1_P1,
+        catalogTarget,
+        Map.of("retention", "7days", "scope", "namespace-only"));
+    policyApi.detachPolicy(currentCatalogName, NS1_P1, catalogTarget);
+
+    policyApi.dropPolicy(currentCatalogName, NS1_P1);
+  }
+
   @Test
   public void testDropNonExistingPolicy() {
     restCatalog.createNamespace(NS1);

persistence/relational-jdbc-mysql/tests/build.gradle.kts

@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+  alias(libs.plugins.quarkus)
+  id("org.kordamp.gradle.jandex")
+  id("polaris-runtime")
+}
+
+// Mirror `runtime/test-common`'s antlr/Scala exclusions and additionally drop
+// jackson-module-scala: Jackson's ServiceLoader otherwise tries to instantiate
+// DefaultScalaModule during RESTEasy provider init and fails.
+configurations.all {
+  if (name != "checkstyle") {
+    exclude(group = "org.antlr", module = "antlr4-runtime")
+    exclude(group = "org.scala-lang", module = "scala-library")
+    exclude(group = "org.scala-lang", module = "scala-reflect")
+    exclude(group = "com.fasterxml.jackson.module", module = "jackson-module-scala_2.12")
+    exclude(group = "com.fasterxml.jackson.module", module = "jackson-module-scala_2.13")
+  }
+}
+
+dependencies {
+  implementation(platform(libs.quarkus.bom))
+  implementation("io.quarkus:quarkus-rest-jackson")
+  // The MySQL JDBC driver lives only in this test module so it stays off the
+  // production-facing `runtime/service` classpath.
+  implementation("io.quarkus:quarkus-jdbc-mysql")
+  implementation(project(":polaris-relational-jdbc"))
+  implementation(project(":polaris-runtime-service"))
+
+  testImplementation(project(":polaris-runtime-test-common"))
+
+  intTestImplementation("io.quarkus:quarkus-junit")
+  intTestImplementation("io.rest-assured:rest-assured")
+  intTestImplementation(project(":polaris-api-management-model"))
+  intTestImplementation(project(":polaris-tests"))
+  // Reuse the existing `ServerManager` (the PolarisServerManager SPI impl) from
+  // runtime-service test fixtures instead of duplicating it in this module.
+  intTestImplementation(testFixtures(project(":polaris-runtime-service")))
+  intTestImplementation(platform(libs.iceberg.bom))
+  intTestImplementation("org.apache.iceberg:iceberg-api")
+  intTestImplementation("org.apache.iceberg:iceberg-core")
+  // CatalogTests / ViewCatalogTests test fixtures used by Mysql*IT base classes.
+  intTestImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests")
+  intTestImplementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests")
+
+  intTestImplementation(platform(libs.testcontainers.bom))
+  intTestImplementation("org.testcontainers:testcontainers-junit-jupiter")
+  intTestImplementation("org.testcontainers:testcontainers-mysql")
+  intTestImplementation(project(":polaris-container-spec-helper"))
+
+  // RESTEasy Classic (via keycloak-admin-client) provides the jakarta.ws.rs.client.ClientBuilder
+  // SPI used by `PolarisClient`. Must remain `intTestRuntimeOnly` so it does not participate
+  // in Quarkus augmentation (which uses RESTEasy Reactive).
+  intTestRuntimeOnly(libs.keycloak.admin.client)
+}
+
+// `runtime/defaults` keeps the MySQL named datasource off by default; flip it on at Quarkus
+// build time for this test module only. The matching `active=true` runtime override is set
+// by `MysqlRelationalJdbcLifeCycleManagement`.
+quarkus { quarkusBuildProperties.put("quarkus.datasource.mysql.jdbc", "true") }
+
+tasks.named("javadoc") { dependsOn("jandex") }
+
+tasks.withType<Test> {
+  if (System.getenv("AWS_REGION") == null) {
+    environment("AWS_REGION", "us-west-2")
+  }
+  environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret")
+  val apiVersion = System.getenv("DOCKER_API_VERSION") ?: "1.44"
+  systemProperty("api.version", apiVersion)
+  jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED")
+  systemProperty("java.security.manager", "allow")
+  maxParallelForks = 1
+
+  val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs")
+  jvmArgumentProviders.add(
+    CommandLineArgumentProvider {
+      listOf("-Dquarkus.log.file.path=${logsDir.resolve("polaris.log").absolutePath}")
+    }
+  )
+  doFirst {
+    logsDir.deleteRecursively()
+    project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete()
+  }
+}

persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlApplicationIT.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.persistence.relational.jdbc.mysql.tests;
+
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.TestProfile;
+import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest;
+
+@TestProfile(MysqlRelationalJdbcProfile.class)
+@QuarkusIntegrationTest
+public class MysqlApplicationIT extends PolarisApplicationIntegrationTest {}

persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlManagementServiceIT.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.persistence.relational.jdbc.mysql.tests;
+
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.TestProfile;
+import org.apache.polaris.service.it.test.PolarisManagementServiceIntegrationTest;
+
+@TestProfile(MysqlRelationalJdbcProfile.class)
+@QuarkusIntegrationTest
+public class MysqlManagementServiceIT extends PolarisManagementServiceIntegrationTest {}

persistence/relational-jdbc-mysql/tests/src/intTest/java/org/apache/polaris/persistence/relational/jdbc/mysql/tests/MysqlPolicyServiceIT.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.persistence.relational.jdbc.mysql.tests;
+
+import io.quarkus.test.junit.QuarkusIntegrationTest;
+import io.quarkus.test.junit.TestProfile;
+import org.apache.polaris.service.it.test.PolarisPolicyServiceIntegrationTest;
+
+@TestProfile(MysqlRelationalJdbcProfile.class)
+@QuarkusIntegrationTest
+public class MysqlPolicyServiceIT extends PolarisPolicyServiceIntegrationTest {}