backport Added extensions and Rate limiting options (#314)

.github/workflows/release.yml

@@ -0,0 +1,60 @@
+name: Release to Maven Central
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "The branch to use to release from."
+        required: true
+        default: "main"
+jobs:
+  release:
+    name: Release to Maven Central
+    runs-on: ubuntu-22.04
+
+    steps:
+    - name: Checkout source code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        ref: ${{ github.event.inputs.branch }}
+        # We need a personal access token to be able to push to a protected branch
+        token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+
+    - name: Set up JDK
+      uses: actions/setup-java@v2
+      with:
+        distribution: 'adopt'
+        java-version: '8'
+        java-package: jdk
+        server-id: sonatype-nexus-staging # Value of the distributionManagement/repository/id field of the pom.xml
+        server-username: SONATYPE_USERNAME # env variable for username in deploy
+        server-password: SONATYPE_PASSWORD # env variable for token in deploy
+        # only signed artifacts will be released to maven central. this sets up things for the maven-gpg-plugin
+        gpg-private-key: ${{ secrets.HCOM_GPG_PRIVATE_KEY }} # Value of the GPG private key to import
+        gpg-passphrase: GPG_PASSPHRASE # env variable for GPG private key passphrase
+        # this creates a settings.xml with the following server
+        settings-path: ${{ github.workspace }}
+
+    - name: Configure Git User
+      run: |
+        git config user.email "[email protected]"
+        git config user.name "eg-oss-ci"
+
+    - name: Install Local Dependencies
+      run: |
+        mvn install:install-file -Dfile=lib/aws-glue-datacatalog-hive-client-1.10.0-WD.pom -DpomFile=lib/aws-glue-datacatalog-hive-client-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/shims-1.10.0-WD.pom -DpomFile=lib/shims-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/shims-common-1.10.0-WD.jar -DpomFile=lib/shims-common-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/hive2-shims-1.10.0-WD.jar -DpomFile=lib/hive2-shims-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/spark-hive-shims-1.10.0-WD.jar -DpomFile=lib/spark-hive-shims-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/shims-loader-1.10.0-WD.jar -DpomFile=lib/shims-loader-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/aws-glue-datacatalog-client-common-1.10.0-WD.jar -DpomFile=lib/aws-glue-datacatalog-client-common-1.10.0-WD.pom
+        mvn install:install-file -Dfile=lib/aws-glue-datacatalog-hive2-client-1.10.0-WD.jar -DpomFile=lib/aws-glue-datacatalog-hive2-client-1.10.0-WD.pom
+
+    - name: Run Maven Targets
+      # we are skipping the tests for releases, run a build step first.
+      run: mvn release:prepare release:perform --settings $GITHUB_WORKSPACE/settings.xml -DskipTests -Darguments=-DskipTests --activate-profiles sonatype-oss-release-github-actions --batch-mode --show-version --no-transfer-progress
+      env:
+        SONATYPE_PASSWORD: ${{ secrets.HCOM_SONATYPE_PASSWORD }}
+        SONATYPE_USERNAME: ${{ secrets.HCOM_SONATYPE_USERNAME }}
+        GPG_PASSPHRASE: ${{secrets.HCOM_GPG_PRIVATE_KEY_PASSPHRASE}}

CHANGELOG.md

@@ -13,7 +13,15 @@
 * `lombok` - `1.18.24`.
 * `jakarta` - `6.0.0`.
 * `apache-commons` - `3.12.0`.
-## [3.12.0] - 2023-02-08
+## [3.13.0] - 2024-04-19
+### Added
+- Added `waggle-dance-extensions` module. See [extensions README](waggle-dance-extensions/README.md.)
+- Added support to enable Rate Limiting in Waggle Dance.
+### Changed
+- Changed and added some log messages for better tracking of calls.
+- Changed Integration Test WaggleDanceRunner to allow for reuse.
+
+## [3.12.0] - 2024-02-08
 ### Added
 - Added optional `primary-meta-store.read-only-remote-meta-store-uris` config to allow traffic to be diverted based on calls made. See README.md.
 

README.md

@@ -112,6 +112,8 @@ The table below describes all the available configuration values for Waggle Danc
 | `status-polling-delay-time-unit`  | No         | Controls the delay time unit. Default is `MINUTES` . |
 | `configuration-properties`        | No         | Map of Hive properties that will be added to the HiveConf used when creating the Thrift clients (they will be shared among all the clients). |
 
+Extensions (for instance Rate Limiting) are described here: [waggle-dance-extensions/README.md](waggle-dance-extensions/README.md)
+
 ### Federation
 
 Federation config is by default located in: `$WAGGLE_DANCE_HOME/conf/waggle-dance-federation.yml`.
@@ -577,4 +579,4 @@ The Waggle Dance logo uses the [Beetype Filled font](http://www.1001fonts.com/be
 ## Legal
 This project is available under the [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0.html).
 
-Copyright 2016-2019 Expedia, Inc.
+Copyright 2016-2024 Expedia, Inc.

pom.xml

@@ -24,6 +24,7 @@
   </scm>
 
   <modules>
+    <module>waggle-dance-extensions</module>
     <module>waggle-dance-core</module>
     <module>waggle-dance-api</module>
     <module>waggle-dance-rest</module>

waggle-dance-boot/pom.xml

@@ -25,6 +25,11 @@
       <artifactId>waggle-dance-rest</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.hotels</groupId>
+      <artifactId>waggle-dance-extensions</artifactId>
+      <version>${project.version}</version>
+    </dependency>
   </dependencies>
 
   <build>

waggle-dance-core/pom.xml

@@ -140,6 +140,22 @@
     <dependency>
       <groupId>org.apache.hive</groupId>
       <artifactId>hive-exec</artifactId>
+      <version>${hive.version}</version>
+      <classifier>core</classifier>
+      <exclusions>
+        <exclusion>
+          <groupId>log4j</groupId>
+          <artifactId>log4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.pentaho</groupId>
+          <artifactId>pentaho-aggdesigner-algorithm</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.calcite.avatica</groupId>
+          <artifactId>avatica</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
@@ -154,6 +170,22 @@
           <groupId>org.eclipse.jetty</groupId>
           <artifactId>jetty-runner</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.eclipse.jetty.aggregate</groupId>
+          <artifactId>jetty-all</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.pentaho</groupId>
+          <artifactId>pentaho-aggdesigner-algorithm</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>jdk.tools</groupId>
+          <artifactId>jdk.tools</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>hive-exec</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
@@ -252,6 +284,12 @@
       <groupId>com.hotels</groupId>
       <artifactId>beeju</artifactId>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>hive-exec</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>fm.last.commons</groupId>

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/WaggleDance.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2023 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIface.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2019 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/CloseableThriftHiveMetastoreIfaceClientFactory.java

@@ -32,7 +32,7 @@
 import com.hotels.hcommon.hive.metastore.conf.HiveConfFactory;
 import com.hotels.hcommon.hive.metastore.util.MetaStoreUriNormaliser;
 
-public class CloseableThriftHiveMetastoreIfaceClientFactory {
+public class CloseableThriftHiveMetastoreIfaceClientFactory implements ThriftClientFactory {
 
   private static final int DEFAULT_CLIENT_FACTORY_RECONNECTION_RETRY = 3;
   private final TunnelingMetaStoreClientFactory tunnelingMetaStoreClientFactory;

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/ThriftClientFactory.java

@@ -0,0 +1,27 @@
+/**
+ * Copyright (C) 2016-2024 Expedia, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.hotels.bdp.waggledance.client;
+
+import com.hotels.bdp.waggledance.api.model.AbstractMetaStore;
+
+public interface ThriftClientFactory {
+
+  /**
+   * @param metaStore (configuration object)
+   * @return client that will be used to query the metaStore.
+   */
+  public CloseableThriftHiveMetastoreIface newInstance(AbstractMetaStore metaStore);
+}

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/ThriftMetastoreClientManager.java

@@ -54,7 +54,7 @@ void open(HiveUgiArgs ugiArgs) {
 
     for (int attempt = 0; !isConnected && (attempt < retries); ++attempt) {
       for (URI store : metastoreUris) {
-        log.info("Trying to connect to metastore with URI {}", store);
+        log.debug("Trying to connect to metastore with URI {}", store);
         transport = new TSocket(store.getHost(), store.getPort(), clientSocketTimeout, connectionTimeout);
         if (useFramedTransport) {
           transport = new TFramedTransport(transport);

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/tunnelling/TunnelableFactorySupplier.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2019 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/client/tunnelling/TunnelingMetaStoreClientFactory.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2023 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/conf/WaggleDanceConfiguration.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2023 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/context/CommonBeans.java

@@ -19,11 +19,13 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.context.annotation.Bean;
 
 import com.hotels.bdp.waggledance.client.CloseableThriftHiveMetastoreIfaceClientFactory;
 import com.hotels.bdp.waggledance.client.DefaultMetaStoreClientFactory;
 import com.hotels.bdp.waggledance.client.SplitTrafficMetastoreClientFactory;
+import com.hotels.bdp.waggledance.client.ThriftClientFactory;
 import com.hotels.bdp.waggledance.client.tunnelling.TunnelingMetaStoreClientFactory;
 import com.hotels.bdp.waggledance.conf.WaggleDanceConfiguration;
 import com.hotels.bdp.waggledance.mapping.model.ASTQueryMapping;
@@ -60,13 +62,21 @@ public SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory() {
     return new SplitTrafficMetastoreClientFactory();
   }
 
+
   @Bean
-  public CloseableThriftHiveMetastoreIfaceClientFactory metaStoreClientFactory(
+  public ThriftClientFactory defaultWaggleDanceClientFactory(
       WaggleDanceConfiguration waggleDanceConfiguration, SplitTrafficMetastoreClientFactory splitTrafficMetaStoreClientFactory) {
     return new CloseableThriftHiveMetastoreIfaceClientFactory(new TunnelingMetaStoreClientFactory(),
         new DefaultMetaStoreClientFactory(), waggleDanceConfiguration, splitTrafficMetaStoreClientFactory);
   }
 
+  //Only load when no other beans with this name can be found.
+  @ConditionalOnMissingBean
+  @Bean
+  public ThriftClientFactory thriftClientFactory(ThriftClientFactory defaultWaggleDanceClientFactory) {
+    return defaultWaggleDanceClientFactory;
+  }
+
   @Bean
   public QueryMapping queryMapping() {
     return ASTQueryMapping.INSTANCE;

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2023 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -33,7 +33,7 @@
 import com.hotels.bdp.waggledance.api.model.AbstractMetaStore;
 import com.hotels.bdp.waggledance.api.model.DatabaseResolution;
 import com.hotels.bdp.waggledance.client.CloseableThriftHiveMetastoreIface;
-import com.hotels.bdp.waggledance.client.CloseableThriftHiveMetastoreIfaceClientFactory;
+import com.hotels.bdp.waggledance.client.ThriftClientFactory;
 import com.hotels.bdp.waggledance.conf.WaggleDanceConfiguration;
 import com.hotels.bdp.waggledance.mapping.service.MetaStoreMappingFactory;
 import com.hotels.bdp.waggledance.mapping.service.PrefixNamingStrategy;
@@ -45,24 +45,24 @@
 public class MetaStoreMappingFactoryImpl implements MetaStoreMappingFactory {
   private final WaggleDanceConfiguration waggleDanceConfiguration;
   private final PrefixNamingStrategy prefixNamingStrategy;
-  private final CloseableThriftHiveMetastoreIfaceClientFactory metaStoreClientFactory;
+  private final ThriftClientFactory thriftClientFactory;
   private final AccessControlHandlerFactory accessControlHandlerFactory;
 
   @Autowired
   public MetaStoreMappingFactoryImpl(
-          WaggleDanceConfiguration waggleDanceConfiguration,
-          PrefixNamingStrategy prefixNamingStrategy,
-          CloseableThriftHiveMetastoreIfaceClientFactory metaStoreClientFactory,
-          AccessControlHandlerFactory accessControlHandlerFactory) {
+      WaggleDanceConfiguration waggleDanceConfiguration,
+      PrefixNamingStrategy prefixNamingStrategy,
+      ThriftClientFactory thriftClientFactory,
+      AccessControlHandlerFactory accessControlHandlerFactory) {
     this.waggleDanceConfiguration = waggleDanceConfiguration;
     this.prefixNamingStrategy = prefixNamingStrategy;
-    this.metaStoreClientFactory = metaStoreClientFactory;
+    this.thriftClientFactory = thriftClientFactory;
     this.accessControlHandlerFactory = accessControlHandlerFactory;
   }
 
   private CloseableThriftHiveMetastoreIface createClient(AbstractMetaStore metaStore) {
     try {
-      return metaStoreClientFactory.newInstance(metaStore);
+      return thriftClientFactory.newInstance(metaStore);
     } catch (Exception e) {
       log.error("Can't create a client for metastore '{}':", metaStore.getName(), e);
       return newUnreachableMetastoreClient(metaStore);

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/ExceptionWrappingHMSHandler.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016-2023 Expedia, Inc.
+ * Copyright (C) 2016-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,6 +22,7 @@
 import java.lang.reflect.UndeclaredThrowableException;
 import java.util.Arrays;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.hive.metastore.IHMSHandler;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 
@@ -33,10 +34,12 @@
 public class ExceptionWrappingHMSHandler implements InvocationHandler {
 
   private final IHMSHandler baseHandler;
+  private String user = "";
 
   public static IHMSHandler newProxyInstance(IHMSHandler baseHandler) {
-    return (IHMSHandler) Proxy.newProxyInstance(ExceptionWrappingHMSHandler.class.getClassLoader(),
-        new Class[] { IHMSHandler.class }, new ExceptionWrappingHMSHandler(baseHandler));
+    return (IHMSHandler) Proxy
+        .newProxyInstance(ExceptionWrappingHMSHandler.class.getClassLoader(), new Class[] { IHMSHandler.class },
+            new ExceptionWrappingHMSHandler(baseHandler));
   }
 
   public ExceptionWrappingHMSHandler(IHMSHandler baseHandler) {
@@ -45,7 +48,13 @@ public ExceptionWrappingHMSHandler(IHMSHandler baseHandler) {
 
   @Override
   public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+    if (method.getName().equals("set_ugi")) {
+      user = (String) args[0];
+    }
     try {
+      log
+          .info("WD Audit:[User:{}, method:{}, args:{}]", user, method.getName(),
+              StringUtils.left(Arrays.toString(args), 256));
       return method.invoke(baseHandler, args);
     } catch (InvocationTargetException e) {
       Throwable cause = e.getCause();

waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java

@@ -366,9 +366,9 @@ public void create_database(Database database)
   @Override
   @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
   public Database get_database(String name) throws NoSuchObjectException, MetaException, TException {
-    log.info("Fetching database {}", name);
+    log.debug("Fetching database {}", name);
     DatabaseMapping mapping = databaseMappingService.databaseMapping(name);
-    log.info("Mapping is '{}'", mapping.getDatabasePrefix());
+    log.debug("Mapping is '{}'", mapping.getDatabasePrefix());
     Database result = mapping.getClient().get_database(mapping.transformInboundDatabaseName(name));
     return mapping.transformOutboundDatabase(mapping.getMetastoreFilter().filterDatabase(result));
   }