From c4b234b445500759c80978ce445c2f1c74a29919 Mon Sep 17 00:00:00 2001 From: wangd Date: Mon, 19 May 2025 08:28:43 +0800 Subject: [PATCH] [Iceberg]Prefix table (loaded from Hive Catalog) with catalog name --- .../presto/iceberg/IcebergHiveMetadata.java | 5 +- .../iceberg/IcebergHiveMetadataFactory.java | 4 ++ .../facebook/presto/iceberg/IcebergUtil.java | 46 ++++++++++++------- .../iceberg/IcebergDistributedTestBase.java | 17 ++++++- .../iceberg/TestIcebergTableVersion.java | 12 ++--- .../TestIcebergDistributedOnS3Hadoop.java | 3 +- .../hive/TestIcebergDistributedHive.java | 2 + .../hive/TestIcebergHiveStatistics.java | 2 + .../iceberg/hive/TestIcebergSmokeHive.java | 4 ++ .../TestRenameTableOnFragileFileSystem.java | 2 + .../TestRemoveOrphanFilesProcedureHive.java | 2 + 11 files changed, 73 insertions(+), 26 deletions(-) diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java index 9b237a0ee6440..8742fe4a7dba3 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadata.java @@ -154,6 +154,7 @@ public class IcebergHiveMetadata { public static final int MAXIMUM_PER_QUERY_TABLE_CACHE_SIZE = 1000; + private final IcebergCatalogName catalogName; private final ExtendedHiveMetastore metastore; private final HdfsEnvironment hdfsEnvironment; private final DateTimeZone timeZone = DateTimeZone.forTimeZone(TimeZone.getTimeZone(ZoneId.of(TimeZone.getDefault().getID()))); @@ -162,6 +163,7 @@ public class IcebergHiveMetadata private final ManifestFileCache manifestFileCache; public IcebergHiveMetadata( + IcebergCatalogName catalogName, ExtendedHiveMetastore metastore, HdfsEnvironment hdfsEnvironment, TypeManager typeManager, @@ -176,6 +178,7 @@ public IcebergHiveMetadata( IcebergTableProperties tableProperties) { super(typeManager, functionResolution, rowExpressionService, commitTaskCodec, nodeVersion, filterStatsCalculatorService, statisticsFileCache, tableProperties); + this.catalogName = requireNonNull(catalogName, "catalogName is null"); this.metastore = requireNonNull(metastore, "metastore is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.hiveTableOeprationsConfig = requireNonNull(hiveTableOeprationsConfig, "hiveTableOperationsConfig is null"); @@ -204,7 +207,7 @@ public boolean schemaExists(ConnectorSession session, String schemaName) @Override protected org.apache.iceberg.Table getRawIcebergTable(ConnectorSession session, SchemaTableName schemaTableName) { - return getHiveIcebergTable(metastore, hdfsEnvironment, hiveTableOeprationsConfig, manifestFileCache, session, schemaTableName); + return getHiveIcebergTable(metastore, hdfsEnvironment, hiveTableOeprationsConfig, manifestFileCache, session, catalogName, schemaTableName); } @Override diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java index 9747146392fb7..2f8026bfba2d2 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergHiveMetadataFactory.java @@ -31,6 +31,7 @@ public class IcebergHiveMetadataFactory implements IcebergMetadataFactory { + final IcebergCatalogName catalogName; final ExtendedHiveMetastore metastore; final HdfsEnvironment hdfsEnvironment; final TypeManager typeManager; @@ -46,6 +47,7 @@ public class IcebergHiveMetadataFactory @Inject public IcebergHiveMetadataFactory( + IcebergCatalogName catalogName, ExtendedHiveMetastore metastore, HdfsEnvironment hdfsEnvironment, TypeManager typeManager, @@ -59,6 +61,7 @@ public IcebergHiveMetadataFactory( ManifestFileCache manifestFileCache, IcebergTableProperties tableProperties) { + this.catalogName = requireNonNull(catalogName, "catalogName is null"); this.metastore = requireNonNull(metastore, "metastore is null"); this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); @@ -76,6 +79,7 @@ public IcebergHiveMetadataFactory( public ConnectorMetadata create() { return new IcebergHiveMetadata( + catalogName, metastore, hdfsEnvironment, typeManager, diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java index d925a37db40bd..dce664827c4a9 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergUtil.java @@ -71,6 +71,7 @@ import org.apache.iceberg.TableOperations; import org.apache.iceberg.TableScan; import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.hive.HiveSchemaUtil; @@ -101,7 +102,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -211,7 +211,6 @@ public final class IcebergUtil { - private static final Pattern SIMPLE_NAME = Pattern.compile("[a-z][a-z0-9]*"); private static final Logger log = Logger.get(IcebergUtil.class); public static final int MIN_FORMAT_VERSION_FOR_DELETE = 2; @@ -246,7 +245,7 @@ public static Table getShallowWrappedIcebergTable(Schema schema, PartitionSpec s return new PrestoIcebergTableForMetricsConfig(schema, spec, properties, sortOrder); } - public static Table getHiveIcebergTable(ExtendedHiveMetastore metastore, HdfsEnvironment hdfsEnvironment, IcebergHiveTableOperationsConfig config, ManifestFileCache manifestFileCache, ConnectorSession session, SchemaTableName table) + public static Table getHiveIcebergTable(ExtendedHiveMetastore metastore, HdfsEnvironment hdfsEnvironment, IcebergHiveTableOperationsConfig config, ManifestFileCache manifestFileCache, ConnectorSession session, IcebergCatalogName catalogName, SchemaTableName table) { HdfsContext hdfsContext = new HdfsContext(session, table.getSchemaName(), table.getTableName()); TableOperations operations = new HiveTableOperations( @@ -258,7 +257,7 @@ public static Table getHiveIcebergTable(ExtendedHiveMetastore metastore, HdfsEnv manifestFileCache, table.getSchemaName(), table.getTableName()); - return new BaseTable(operations, quotedTableName(table)); + return new BaseTable(operations, fullTableName(catalogName.getCatalogName(), TableIdentifier.of(table.getSchemaName(), table.getTableName()))); } public static Table getNativeIcebergTable(IcebergNativeCatalogFactory catalogFactory, ConnectorSession session, SchemaTableName table) @@ -411,19 +410,6 @@ public static Optional getViewComment(View view) return Optional.ofNullable(view.properties().get(TABLE_COMMENT)); } - private static String quotedTableName(SchemaTableName name) - { - return quotedName(name.getSchemaName()) + "." + quotedName(name.getTableName()); - } - - private static String quotedName(String name) - { - if (SIMPLE_NAME.matcher(name).matches()) { - return name; - } - return '"' + name.replace("\"", "\"\"") + '"'; - } - public static TableScan getTableScan(TupleDomain predicates, Optional snapshotId, Table icebergTable) { Expression expression = ExpressionConverter.toIcebergExpression(predicates); @@ -1329,4 +1315,30 @@ public static DataSize getTargetSplitSize(ConnectorSession session, Scan