Add support for MySQL Metastore

gradle/libs.versions.toml

@@ -90,6 +90,7 @@ opentelemetry-bom = { module = "io.opentelemetry:opentelemetry-bom", version = "
 opentelemetry-semconv = { module = "io.opentelemetry.semconv:opentelemetry-semconv", version = "1.37.0" }
 picocli = { module = "info.picocli:picocli-codegen", version.ref = "picocli" }
 picocli-codegen = { module = "info.picocli:picocli-codegen", version.ref = "picocli" }
+mysql-connector-j = { module = "com.mysql:mysql-connector-j", version = "8.0.33" }
 postgresql = { module = "org.postgresql:postgresql", version = "42.7.8" }
 prometheus-metrics-exporter-servlet-jakarta = { module = "io.prometheus:prometheus-metrics-exporter-servlet-jakarta", version = "1.4.1" }
 quarkus-bom = { module = "io.quarkus.platform:quarkus-bom", version.ref = "quarkus" }

persistence/relational-jdbc/build.gradle.kts

@@ -35,6 +35,7 @@ dependencies {
 
   implementation(libs.smallrye.common.annotation) // @Identifier
   implementation(libs.postgresql)
+  implementation(libs.mysql.connector.j) 
 
   compileOnly(project(":polaris-immutables"))
   annotationProcessor(project(":polaris-immutables", configuration = "processor"))

persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/DatabaseType.java

@@ -27,7 +27,8 @@
 
 public enum DatabaseType {
   POSTGRES("postgres"),
-  H2("h2");
+  H2("h2"),
+  MYSQL("mysql");
 
   private final String displayName; // Store the user-friendly name
 
@@ -44,6 +45,7 @@ public static DatabaseType fromDisplayName(String displayName) {
     return switch (displayName.toLowerCase(Locale.ROOT)) {
       case "h2" -> DatabaseType.H2;
       case "postgresql" -> DatabaseType.POSTGRES;
+      case "mysql" -> DatabaseType.MYSQL;
       default -> throw new IllegalStateException("Unsupported DatabaseType: '" + displayName + "'");
     };
   }

persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/QueryGenerator.java

@@ -250,7 +250,8 @@ static QueryFragment generateWhereClause(
 
   @VisibleForTesting
   static PreparedQuery generateVersionQuery() {
-    return new PreparedQuery("SELECT version_value FROM POLARIS_SCHEMA.VERSION", List.of());
+    String tableName = getFullyQualifiedTableName("VERSION");
+    return new PreparedQuery("SELECT version_value FROM " + tableName, List.of());
   }
 
   /**

persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/Converter.java

@@ -50,4 +50,18 @@ default PGobject toJsonbPGobject(String props) {
       throw new RuntimeException(e);
     }
   }
+
+  /**
+   * Creates a JSON object based on the database type.
+   */
+  default Object toJsonObject(String props, DatabaseType databaseType) {
+    switch (databaseType) {
+      case POSTGRES:
+        return toJsonbPGobject(props);
+      case MYSQL:
+      case H2:
+      default:
+        return props; 
+    }
+  }
 }

persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEntity.java

@@ -216,13 +216,8 @@ public Map<String, Object> toMap(DatabaseType databaseType) {
     map.put("purge_timestamp", this.getPurgeTimestamp());
     map.put("to_purge_timestamp", this.getToPurgeTimestamp());
     map.put("last_update_timestamp", this.getLastUpdateTimestamp());
-    if (databaseType.equals(DatabaseType.POSTGRES)) {
-      map.put("properties", toJsonbPGobject(this.getProperties()));
-      map.put("internal_properties", toJsonbPGobject(this.getInternalProperties()));
-    } else {
-      map.put("properties", this.getProperties());
-      map.put("internal_properties", this.getInternalProperties());
-    }
+    map.put("properties", toJsonObject(this.getProperties(), databaseType));
+    map.put("internal_properties", toJsonObject(this.getInternalProperties(), databaseType));
     map.put("grant_records_version", this.getGrantRecordsVersion());
     map.put("location_without_scheme", this.getLocationWithoutScheme());
     return map;

persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelEvent.java

@@ -102,11 +102,7 @@ default Map<String, Object> toMap(DatabaseType databaseType) {
     map.put("principal_name", getPrincipalName());
     map.put("resource_type", getResourceType().toString());
     map.put("resource_identifier", getResourceIdentifier());
-    if (databaseType.equals(DatabaseType.POSTGRES)) {
-      map.put("additional_properties", toJsonbPGobject(getAdditionalProperties()));
-    } else {
-      map.put("additional_properties", getAdditionalProperties());
-    }
+    map.put("additional_properties", toJsonObject(getAdditionalProperties(), databaseType));
     return map;
   }
 

persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/models/ModelPolicyMappingRecord.java

@@ -175,11 +175,7 @@ public Map<String, Object> toMap(DatabaseType databaseType) {
     map.put("policy_type_code", policyTypeCode);
     map.put("policy_catalog_id", policyCatalogId);
     map.put("policy_id", policyId);
-    if (databaseType.equals(DatabaseType.POSTGRES)) {
-      map.put("parameters", toJsonbPGobject(this.getParameters()));
-    } else {
-      map.put("parameters", this.getParameters());
-    }
+    map.put("parameters", toJsonObject(this.getParameters(), databaseType));
     return map;
   }
 }

persistence/relational-jdbc/src/main/resources/mysql/schema-v1.sql

@@ -0,0 +1,104 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file--
+--  distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"). You may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--  http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Changes from v2:
+--  * Added `events` table
+
+CREATE DATABASE IF NOT EXISTS POLARIS_SCHEMA;
+USE POLARIS_SCHEMA;
+
+CREATE TABLE IF NOT EXISTS version (
+    version_key VARCHAR(255) PRIMARY KEY,
+    version_value INTEGER NOT NULL
+);
+INSERT INTO version (version_key, version_value)
+VALUES ('version', 3)
+ON DUPLICATE KEY UPDATE version_value = VALUES(version_value);
+
+CREATE TABLE IF NOT EXISTS entities (
+    realm_id VARCHAR(255) NOT NULL,
+    catalog_id BIGINT NOT NULL,
+    id BIGINT NOT NULL,
+    parent_id BIGINT NOT NULL,
+    name VARCHAR(255) NOT NULL,
+    entity_version INT NOT NULL,
+    type_code INT NOT NULL,
+    sub_type_code INT NOT NULL,
+    create_timestamp BIGINT NOT NULL,
+    drop_timestamp BIGINT NOT NULL,
+    purge_timestamp BIGINT NOT NULL,
+    to_purge_timestamp BIGINT NOT NULL,
+    last_update_timestamp BIGINT NOT NULL,
+    properties JSON NOT NULL DEFAULT ('{}'),
+    internal_properties JSON NOT NULL DEFAULT ('{}'),
+    grant_records_version INT NOT NULL,
+    location_without_scheme TEXT,
+    PRIMARY KEY (realm_id, id),
+    UNIQUE KEY constraint_name (realm_id, catalog_id, parent_id, type_code, name)
+);
+
+-- TODO: create indexes based on all query pattern.
+CREATE INDEX idx_entities ON entities (realm_id, catalog_id, id);
+CREATE INDEX idx_locations ON entities (realm_id, parent_id, location_without_scheme(255));
+
+CREATE TABLE IF NOT EXISTS grant_records (
+    realm_id VARCHAR(255) NOT NULL,
+    securable_catalog_id BIGINT NOT NULL,
+    securable_id BIGINT NOT NULL,
+    grantee_catalog_id BIGINT NOT NULL,
+    grantee_id BIGINT NOT NULL,
+    privilege_code INTEGER,
+    PRIMARY KEY (realm_id, securable_catalog_id, securable_id, grantee_catalog_id, grantee_id, privilege_code)
+);
+
+CREATE TABLE IF NOT EXISTS principal_authentication_data (
+    realm_id VARCHAR(255) NOT NULL,
+    principal_id BIGINT NOT NULL,
+    principal_client_id VARCHAR(255) NOT NULL,
+    main_secret_hash VARCHAR(255) NOT NULL,
+    secondary_secret_hash VARCHAR(255) NOT NULL,
+    secret_salt VARCHAR(255) NOT NULL,
+    PRIMARY KEY (realm_id, principal_client_id)
+);
+
+CREATE TABLE IF NOT EXISTS policy_mapping_record (
+    realm_id VARCHAR(255) NOT NULL,
+    target_catalog_id BIGINT NOT NULL,
+    target_id BIGINT NOT NULL,
+    policy_type_code INTEGER NOT NULL,
+    policy_catalog_id BIGINT NOT NULL,
+    policy_id BIGINT NOT NULL,
+    parameters JSON NOT NULL DEFAULT ('{}'),
+    PRIMARY KEY (realm_id, target_catalog_id, target_id, policy_type_code, policy_catalog_id, policy_id)
+);
+
+CREATE INDEX idx_policy_mapping_record ON policy_mapping_record (realm_id, policy_type_code, policy_catalog_id, policy_id, target_catalog_id, target_id);
+
+CREATE TABLE IF NOT EXISTS events (
+    realm_id VARCHAR(255) NOT NULL,
+    catalog_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(255) NOT NULL,
+    request_id VARCHAR(255),
+    event_type VARCHAR(255) NOT NULL,
+    timestamp_ms BIGINT NOT NULL,
+    principal_name VARCHAR(255),
+    resource_type VARCHAR(255) NOT NULL,
+    resource_identifier VARCHAR(255) NOT NULL,
+    additional_properties JSON NOT NULL DEFAULT ('{}'),
+    PRIMARY KEY (event_id)
+);

persistence/relational-jdbc/src/main/resources/mysql/schema-v2.sql

@@ -0,0 +1,104 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file--
+--  distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"). You may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--  http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Changes from v2:
+--  * Added `events` table
+
+CREATE DATABASE IF NOT EXISTS POLARIS_SCHEMA;
+USE POLARIS_SCHEMA;
+
+CREATE TABLE IF NOT EXISTS version (
+    version_key VARCHAR(255) PRIMARY KEY,
+    version_value INTEGER NOT NULL
+);
+INSERT INTO version (version_key, version_value)
+VALUES ('version', 3)
+ON DUPLICATE KEY UPDATE version_value = VALUES(version_value);
+
+CREATE TABLE IF NOT EXISTS entities (
+    realm_id VARCHAR(255) NOT NULL,
+    catalog_id BIGINT NOT NULL,
+    id BIGINT NOT NULL,
+    parent_id BIGINT NOT NULL,
+    name VARCHAR(255) NOT NULL,
+    entity_version INT NOT NULL,
+    type_code INT NOT NULL,
+    sub_type_code INT NOT NULL,
+    create_timestamp BIGINT NOT NULL,
+    drop_timestamp BIGINT NOT NULL,
+    purge_timestamp BIGINT NOT NULL,
+    to_purge_timestamp BIGINT NOT NULL,
+    last_update_timestamp BIGINT NOT NULL,
+    properties JSON NOT NULL DEFAULT ('{}'),
+    internal_properties JSON NOT NULL DEFAULT ('{}'),
+    grant_records_version INT NOT NULL,
+    location_without_scheme TEXT,
+    PRIMARY KEY (realm_id, id),
+    UNIQUE KEY constraint_name (realm_id, catalog_id, parent_id, type_code, name)
+);
+
+-- TODO: create indexes based on all query pattern.
+CREATE INDEX idx_entities ON entities (realm_id, catalog_id, id);
+CREATE INDEX idx_locations ON entities (realm_id, parent_id, location_without_scheme(255));
+
+CREATE TABLE IF NOT EXISTS grant_records (
+    realm_id VARCHAR(255) NOT NULL,
+    securable_catalog_id BIGINT NOT NULL,
+    securable_id BIGINT NOT NULL,
+    grantee_catalog_id BIGINT NOT NULL,
+    grantee_id BIGINT NOT NULL,
+    privilege_code INTEGER,
+    PRIMARY KEY (realm_id, securable_catalog_id, securable_id, grantee_catalog_id, grantee_id, privilege_code)
+);
+
+CREATE TABLE IF NOT EXISTS principal_authentication_data (
+    realm_id VARCHAR(255) NOT NULL,
+    principal_id BIGINT NOT NULL,
+    principal_client_id VARCHAR(255) NOT NULL,
+    main_secret_hash VARCHAR(255) NOT NULL,
+    secondary_secret_hash VARCHAR(255) NOT NULL,
+    secret_salt VARCHAR(255) NOT NULL,
+    PRIMARY KEY (realm_id, principal_client_id)
+);
+
+CREATE TABLE IF NOT EXISTS policy_mapping_record (
+    realm_id VARCHAR(255) NOT NULL,
+    target_catalog_id BIGINT NOT NULL,
+    target_id BIGINT NOT NULL,
+    policy_type_code INTEGER NOT NULL,
+    policy_catalog_id BIGINT NOT NULL,
+    policy_id BIGINT NOT NULL,
+    parameters JSON NOT NULL DEFAULT ('{}'),
+    PRIMARY KEY (realm_id, target_catalog_id, target_id, policy_type_code, policy_catalog_id, policy_id)
+);
+
+CREATE INDEX idx_policy_mapping_record ON policy_mapping_record (realm_id, policy_type_code, policy_catalog_id, policy_id, target_catalog_id, target_id);
+
+CREATE TABLE IF NOT EXISTS events (
+    realm_id VARCHAR(255) NOT NULL,
+    catalog_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(255) NOT NULL,
+    request_id VARCHAR(255),
+    event_type VARCHAR(255) NOT NULL,
+    timestamp_ms BIGINT NOT NULL,
+    principal_name VARCHAR(255),
+    resource_type VARCHAR(255) NOT NULL,
+    resource_identifier VARCHAR(255) NOT NULL,
+    additional_properties JSON NOT NULL DEFAULT ('{}'),
+    PRIMARY KEY (event_id)
+);