HIVE-28972: HMS performace degradation post HIVE-28909 for alter query (#5835)

Author: dengzhhu653

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java

@@ -407,6 +407,9 @@ public List<Void> run(List<Partition> input) throws Exception {
               MetastoreConf.getBoolVar(handler.getConf(), MetastoreConf.ConfVars.COLSTATS_RETAIN_ON_COLUMN_REMOVAL);
 
           if (runPartitionMetadataUpdate) {
+            // Don't validate table-level stats for a partitoned table.
+            msdb.alterTable(catName, dbname, name, newt, null);
+
             if (cascade || retainOnColRemoval) {
               parts = msdb.getPartitions(catName, dbname, name, -1);
               for (Partition part : parts) {
@@ -431,8 +434,6 @@ public List<Void> run(List<Partition> input) throws Exception {
               TableName tableName = new TableName(catName, dbname, name);
               msdb.deleteAllPartitionColumnStatistics(tableName, writeIdList);
             }
-            // Don't validate table-level stats for a partitoned table.
-            msdb.alterTable(catName, dbname, name, newt, null);
           } else {
             LOG.warn("Alter table not cascaded to partitions.");
             msdb.alterTable(catName, dbname, name, newt, writeIdList);

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java

@@ -2715,7 +2715,7 @@ private void addPartitionsInternal(String catName, String dbName,
         throw new MetaException("Partition does not belong to target table "
             + dbName + "." + tblName + ": " + part);
       }
-      MPartition mpart = convertToMPart(part, table, true);
+      MPartition mpart = convertToMPart(part, table);
       mParts.add(mpart);
       int now = (int) (System.currentTimeMillis() / 1000);
       List<MPartitionPrivilege> mPartPrivileges = new ArrayList<>();
@@ -2817,7 +2817,7 @@ public boolean addPartitions(String catName, String dbName, String tblName,
         Partition part = iterator.next();
 
         if (isValidPartition(part, partitionKeys, ifNotExists)) {
-          MPartition mpart = convertToMPart(part, table, true);
+          MPartition mpart = convertToMPart(part, table);
           pm.makePersistent(mpart);
           if (tabGrants != null) {
             for (MTablePrivilege tab : tabGrants) {
@@ -3013,10 +3013,9 @@ private MPartition getMPartition(String catName, String dbName, String tableName
    * to the same one as the table's storage descriptor.
    * @param part the partition to convert
    * @param mt the parent table object
-   * @param useTableCD whether to try to use the parent table's column descriptor.
    * @return the model partition object, and null if the input partition is null.
    */
-  private MPartition convertToMPart(Partition part, MTable mt, boolean useTableCD)
+  private MPartition convertToMPart(Partition part, MTable mt)
       throws InvalidObjectException, MetaException {
     // NOTE: we don't set writeId in this method. Write ID is only set after validating the
     //       existing write ID against the caller's valid list.
@@ -3032,8 +3031,7 @@ private MPartition convertToMPart(Partition part, MTable mt, boolean useTableCD)
     // use the parent table's, so we do not create a duplicate column descriptor,
     // thereby saving space
     MStorageDescriptor msd;
-    if (useTableCD &&
-        mt.getSd() != null && mt.getSd().getCD() != null &&
+    if (mt.getSd() != null && mt.getSd().getCD() != null &&
         mt.getSd().getCD().getCols() != null &&
         part.getSd() != null &&
         convertToFieldSchemas(mt.getSd().getCD().getCols()).
@@ -5188,7 +5186,7 @@ private Partition alterPartitionNoTxn(String catName, String dbname,
     catName = normalizeIdentifier(catName);
     name = normalizeIdentifier(name);
     dbname = normalizeIdentifier(dbname);
-    MPartition newp = convertToMPart(newPart, table, false);
+    MPartition newp = convertToMPart(newPart, table);
     MColumnDescriptor oldCD = null;
     MStorageDescriptor oldSD = oldp.getSd();
     if (oldSD != null) {
@@ -5248,9 +5246,6 @@ public Partition alterPartition(String catName, String dbname, String name, List
     Partition result = null;
     try {
       openTransaction();
-      if (newPart.isSetWriteId()) {
-        LOG.warn("Alter partitions with write ID called without transaction information");
-      }
       Ref<MColumnDescriptor> oldCd = new Ref<>();
       result = alterPartitionNoTxn(catName, dbname, name, part_vals, newPart, validWriteIds, oldCd);
       removeUnusedColumnDescriptor(oldCd.t);

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MColumn.java

@@ -62,7 +62,8 @@ public boolean equals(Object other) {
   public MColumn() {
   }
 
-  public MColumn(String name, String type, String comment) {
+  public MColumn(MColumnDescriptor cd, String name, String type, String comment) {
+    this.cd = cd;
     this.name = name;
     this.type = type;
     this.comment = comment;

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/model/MColumnDescriptor.java

@@ -16,9 +16,6 @@
  * limitations under the License.
  */
 
-/**
- *
- */
 package org.apache.hadoop.hive.metastore.model;
 
 import javax.jdo.annotations.NotPersistent;
@@ -42,7 +39,7 @@ public MColumnDescriptor() {}
 
   public MColumnDescriptor(List<MFieldSchema> cols) {
     fields = cols.stream().map(schema ->
-        new MColumn(schema.getName(), schema.getType(), schema.getComment()))
+        new MColumn(this, schema.getName(), schema.getType(), schema.getComment()))
             .collect(Collectors.toList());
   }