HIVE-28775: HiveServer2: introduce a new HA Health Check endpoint on a different p…

common/src/java/org/apache/hadoop/hive/conf/HiveConf.java

@@ -3979,6 +3979,8 @@ public static enum ConfVars {
     HIVE_SERVER2_WEBUI_BIND_HOST("hive.server2.webui.host", "0.0.0.0", "The host address the HiveServer2 WebUI will listen on"),
     HIVE_SERVER2_WEBUI_PORT("hive.server2.webui.port", 10002, "The port the HiveServer2 WebUI will listen on. This can be"
         + "set to 0 or a negative integer to disable the web UI"),
+    HIVE_SERVER2_HEALTH_HA_PORT("hive.server2.health.ha.port", 11002, "The port the HiveServer2 health-ha web app will listen on. This can be"
+        + "set to 0 or a negative integer to disable HS2 health-ha checking"),
     HIVE_SERVER2_WEBUI_MAX_THREADS("hive.server2.webui.max.threads", 50, "The max HiveServer2 WebUI threads"),
     HIVE_SERVER2_WEBUI_USE_SSL("hive.server2.webui.use.ssl", false,
         "Set this to true for using SSL encryption for HiveServer2 WebUI."),

data/conf/hive-site.xml

@@ -358,7 +358,7 @@
 
   <property>
     <name>hive.server2.webui.max.threads</name>
-    <value>4</value>
+    <value>6</value>
   </property>
 
   <property>

data/conf/mr/hive-site.xml

@@ -354,7 +354,7 @@
 
   <property>
     <name>hive.server2.webui.max.threads</name>
-    <value>4</value>
+    <value>6</value>
   </property>
 
   <property>

itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java

@@ -139,6 +139,72 @@ private static void setHAConfigs(Configuration conf) {
     conf.setInt(ConfVars.HIVE_ZOOKEEPER_CONNECTION_MAX_RETRIES.varname, 1);
   }
 
+  @Test(timeout = 60000)
+  public void testHealthCheckHA() throws Exception {
+    String instanceId1 = UUID.randomUUID().toString();
+    miniHS2_1.start(getConfOverlay(instanceId1));
+
+    String leaderURL = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
+    String healthCheckURL = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
+
+    String leaderURLHealthCheckPort = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/leader";
+    String healthCheckURLWebUIPort = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/health-ha/leader";
+
+    assertEquals(true, miniHS2_1.getIsLeaderTestFuture().get());
+    assertEquals(true, miniHS2_1.isLeader());
+
+    assertEquals("true", sendGet(leaderURL));
+    assertEquals("true", sendGet(healthCheckURL));
+
+    assertEquals("Not Found", sendGet(leaderURLHealthCheckPort));
+    assertEquals("Not Found", sendGet(healthCheckURLWebUIPort));
+  }
+
+  @Test(timeout = 60000)
+  public void testHealthCheckHAAuth() throws Exception {
+    hiveConf1.setBoolVar(ConfVars.HIVE_SERVER2_WEBUI_ENABLE_CORS, true);
+    setPamConfs(hiveConf1);
+    PamAuthenticator pamAuthenticator = new TestHS2HttpServerPam.TestPamAuthenticator(hiveConf1);
+    String instanceId1 = UUID.randomUUID().toString();
+    miniHS2_1.setPamAuthenticator(pamAuthenticator);
+    miniHS2_1.start(getSecureConfOverlay(instanceId1));
+
+    String leaderURL = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
+    String healthCheckURL = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
+
+    String leaderURLHealthCheckPort = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/leader";
+    String healthCheckURLWebUIPort = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/health-ha/leader";
+
+    assertEquals(true, miniHS2_1.getIsLeaderTestFuture().get());
+    assertEquals(true, miniHS2_1.isLeader());
+
+    assertEquals("true", sendGet(leaderURL, true, true));
+    assertEquals("true", sendGet(healthCheckURL, true, true));
+
+    try {
+      sendGet(leaderURLHealthCheckPort, true, true);
+    } catch (AssertionError e) {
+      assertTrue(e.getMessage().contains("Not Found"));
+    } catch (Exception e) {
+      fail("Expected AssertionError");
+    }
+
+    assertEquals("Not Found", sendGet(healthCheckURLWebUIPort, true, true));
+    assertEquals("Method Not Allowed", sendDelete(healthCheckURL, true, true));
+    assertEquals("Method Not Allowed", sendDelete(healthCheckURLWebUIPort, true, true));
+
+    try {
+      sendDelete(leaderURLHealthCheckPort, true, true);
+    } catch (AssertionError e) {
+      assertTrue(e.getMessage().contains("Not Found"));
+    } catch (Exception e) {
+      fail("Expected AssertionError");
+    }
+
+    String resp = sendDelete(leaderURL, true, true);
+    assertTrue(resp.contains("Failover successful!"));
+  }
+
   @Test(timeout = 60000)
   public void testActivePassiveHA() throws Exception {
     String instanceId1 = UUID.randomUUID().toString();
@@ -150,10 +216,14 @@ public void testActivePassiveHA() throws Exception {
     assertEquals(true, miniHS2_1.isLeader());
     String url = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
     assertEquals("true", sendGet(url));
+    String healthCheckURL = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
+    assertEquals("true", sendGet(healthCheckURL));
 
     assertEquals(false, miniHS2_2.isLeader());
     url = "http://localhost:" + hiveConf2.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
     assertEquals("false", sendGet(url));
+    healthCheckURL = "http://localhost:" + hiveConf2.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
+    assertEquals("false", sendGet(healthCheckURL));
 
     url = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/peers";
     String resp = sendGet(url);
@@ -192,6 +262,8 @@ public void testActivePassiveHA() throws Exception {
     assertEquals(true, miniHS2_2.isLeader());
     url = "http://localhost:" + hiveConf2.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
     assertEquals("true", sendGet(url));
+    healthCheckURL = "http://localhost:" + hiveConf2.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
+    assertEquals("true", sendGet(healthCheckURL));
 
     while (client.getAll().size() != 1) {
       Thread.sleep(100);
@@ -233,6 +305,8 @@ public void testActivePassiveHA() throws Exception {
     assertEquals(false, miniHS2_1.isLeader());
     url = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
     assertEquals("false", sendGet(url));
+    healthCheckURL = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
+    assertEquals("false", sendGet(healthCheckURL));
 
     while (client.getAll().size() != 2) {
       Thread.sleep(100);
@@ -282,10 +356,14 @@ public void testConnectionActivePassiveHAServiceDiscovery() throws Exception {
     assertEquals(true, miniHS2_1.isLeader());
     String url = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
     assertEquals("true", sendGet(url));
+    String healthCheckURL = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
+    assertEquals("true", sendGet(healthCheckURL));
 
     assertEquals(false, miniHS2_2.isLeader());
     url = "http://localhost:" + hiveConf2.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
     assertEquals("false", sendGet(url));
+    healthCheckURL = "http://localhost:" + hiveConf2.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
+    assertEquals("false", sendGet(healthCheckURL));
 
     // miniHS2_1 will be leader
     String zkConnectString = zkServer.getConnectString();
@@ -347,11 +425,14 @@ public void testManualFailover() throws Exception {
       miniHS2_2.start(confOverlay);
       String url1 = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
       String url2 = "http://localhost:" + hiveConf2.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
+      String healthCheckURL1 = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
+      String healthCheckURL2 = "http://localhost:" + hiveConf2.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
 
       // when we start miniHS2_1 will be leader (sequential start)
       assertEquals(true, miniHS2_1.getIsLeaderTestFuture().get());
       assertEquals(true, miniHS2_1.isLeader());
       assertEquals("true", sendGet(url1, true, true));
+      assertEquals("true", sendGet(healthCheckURL1, true, true));
 
       // trigger failover on miniHS2_1
       String resp = sendDelete(url1, true, true);
@@ -361,11 +442,13 @@ public void testManualFailover() throws Exception {
       assertEquals(true, miniHS2_1.getNotLeaderTestFuture().get());
       assertEquals(false, miniHS2_1.isLeader());
       assertEquals("false", sendGet(url1, true, true));
+      assertEquals("false", sendGet(healthCheckURL1, true, true));
 
       // make sure miniHS2_2 is the new leader
       assertEquals(true, miniHS2_2.getIsLeaderTestFuture().get());
       assertEquals(true, miniHS2_2.isLeader());
       assertEquals("true", sendGet(url2, true, true));
+      assertEquals("true", sendGet(healthCheckURL2, true, true));
 
       // send failover request again to miniHS2_1 and get a failure
       resp = sendDelete(url1, true, true);
@@ -379,8 +462,10 @@ public void testManualFailover() throws Exception {
       assertEquals(true, miniHS2_1.getIsLeaderTestFuture().get());
       assertEquals(true, miniHS2_1.isLeader());
       assertEquals("true", sendGet(url1, true, true));
+      assertEquals("true", sendGet(healthCheckURL1, true, true));
       assertEquals(true, miniHS2_2.getNotLeaderTestFuture().get());
       assertEquals("false", sendGet(url2, true, true));
+      assertEquals("false", sendGet(healthCheckURL2, true, true));
       assertEquals(false, miniHS2_2.isLeader());
     } finally {
       resetFailoverConfs();
@@ -404,10 +489,12 @@ public void testManualFailoverUnauthorized() throws Exception {
       miniHS2_2.start(confOverlay);
 
       String url1 = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
+      String healthCheckURL1 = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
       // when we start miniHS2_1 will be leader (sequential start)
       assertEquals(true, miniHS2_1.getIsLeaderTestFuture().get());
       assertEquals(true, miniHS2_1.isLeader());
       assertEquals("true", sendGet(url1, true));
+      assertEquals("true", sendGet(healthCheckURL1, true));
 
       // trigger failover on miniHS2_1 without authorization header
       assertTrue(sendDelete(url1, false).contains("Unauthorized"));
@@ -439,6 +526,7 @@ public void testNoConnectionOnPassive() throws Exception {
       miniHS2_2.setPamAuthenticator(pamAuthenticator2);
       miniHS2_2.start(confOverlay);
       String url1 = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
+      String healthCheckURL1 = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
       assertEquals(true, miniHS2_1.getIsLeaderTestFuture().get());
       assertEquals(true, miniHS2_1.isLeader());
 
@@ -454,6 +542,9 @@ public void testNoConnectionOnPassive() throws Exception {
         Thread.sleep(100);
       }
 
+      resp = sendDelete(healthCheckURL1, true);
+      assertTrue(resp, resp.contains("Method Not Allowed"));
+
       assertEquals(true, miniHS2_2.getIsLeaderTestFuture().get());
       assertEquals(true, miniHS2_2.isLeader());
 
@@ -496,6 +587,8 @@ public void testClientConnectionsOnFailover() throws Exception {
       miniHS2_2.start(confOverlay);
       String url1 = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
       String url2 = "http://localhost:" + hiveConf2.get(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname) + "/leader";
+      String healthCheckUrl1 = "http://localhost:" + hiveConf1.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
+      String healthCheckUrl2 = "http://localhost:" + hiveConf2.get(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT.varname) + "/health-ha/leader";
       String zkJdbcUrl = miniHS2_1.getJdbcURL();
       String zkConnectString = zkServer.getConnectString();
       assertTrue(zkJdbcUrl.contains(zkConnectString));
@@ -504,6 +597,7 @@ public void testClientConnectionsOnFailover() throws Exception {
       assertEquals(true, miniHS2_1.getIsLeaderTestFuture().get());
       assertEquals(true, miniHS2_1.isLeader());
       assertEquals("true", sendGet(url1, true));
+      assertEquals("true", sendGet(healthCheckUrl1, true));
 
       // before failover, check if we are getting connection from miniHS2_1
       String hs2_1_directUrl = "jdbc:hive2://" + miniHS2_1.getHost() + ":" + miniHS2_1.getBinaryPort() +
@@ -523,15 +617,20 @@ public void testClientConnectionsOnFailover() throws Exception {
         Thread.sleep(100);
       }
 
+      resp = sendDelete(healthCheckUrl1, true);
+      assertTrue(resp.contains("Method Not Allowed"));
+
       // make sure miniHS2_1 is not leader
       assertEquals(true, miniHS2_1.getNotLeaderTestFuture().get());
       assertEquals(false, miniHS2_1.isLeader());
       assertEquals("false", sendGet(url1, true));
+      assertEquals("false", sendGet(healthCheckUrl1, true));
 
       // make sure miniHS2_2 is the new leader
       assertEquals(true, miniHS2_2.getIsLeaderTestFuture().get());
       assertEquals(true, miniHS2_2.isLeader());
       assertEquals("true", sendGet(url2, true));
+      assertEquals("true", sendGet(healthCheckUrl2, true));
 
       // when we make a new connection we should get it from miniHS2_2 this time
       String hs2_2_directUrl = "jdbc:hive2://" + miniHS2_2.getHost() + ":" + miniHS2_2.getHttpPort() +
@@ -555,8 +654,10 @@ public void testClientConnectionsOnFailover() throws Exception {
       assertEquals(true, miniHS2_1.getIsLeaderTestFuture().get());
       assertEquals(true, miniHS2_1.isLeader());
       assertEquals("true", sendGet(url1, true));
+      assertEquals("true", sendGet(healthCheckUrl1, true));
       assertEquals(true, miniHS2_2.getNotLeaderTestFuture().get());
       assertEquals("false", sendGet(url2, true));
+      assertEquals("false", sendGet(healthCheckUrl2, true));
       assertEquals(false, miniHS2_2.isLeader());
       // make sure miniHS2_2 closes all its connections
       while (miniHS2_2.getOpenSessionsCount() != 0) {

itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/AbstractHiveService.java

@@ -36,15 +36,17 @@ public abstract class AbstractHiveService {
   private int binaryPort;
   private int httpPort;
   private int webPort;
+  private int healthHAPort;
   private boolean startedHiveService = false;
   private List<String> addedProperties = new ArrayList<String>();
 
-  public AbstractHiveService(HiveConf hiveConf, String hostname, int binaryPort, int httpPort, int webPort) {
+  AbstractHiveService(HiveConf hiveConf, String hostname, int binaryPort, int httpPort, int webPort, int healthHAPort) {
     this.hiveConf = hiveConf;
     this.hostname = hostname;
     this.binaryPort = binaryPort;
     this.httpPort = httpPort;
     this.webPort = webPort;
+    this.healthHAPort = healthHAPort;
   }
 
   /**
@@ -142,6 +144,10 @@ public int getWebPort() {
     return webPort;
   }
 
+  public int getHealthHAPort() {
+    return healthHAPort;
+  }
+
   public boolean isStarted() {
     return startedHiveService;
   }

itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java

@@ -292,6 +292,8 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM
         (usePortsFromConf ? hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT) : MetaStoreTestUtils
             .findFreePort()),
         (usePortsFromConf ? hiveConf.getIntVar(ConfVars.HIVE_SERVER2_WEBUI_PORT) : MetaStoreTestUtils
+            .findFreePort()),
+        (usePortsFromConf ? hiveConf.getIntVar(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT) : MetaStoreTestUtils
             .findFreePort()));
     hiveConf.setLongVar(ConfVars.HIVE_SERVER2_MAX_START_ATTEMPTS, 3l);
     hiveConf.setTimeVar(ConfVars.HIVE_SERVER2_SLEEP_INTERVAL_BETWEEN_START_ATTEMPTS, 10,
@@ -379,6 +381,7 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM
     hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, getBinaryPort());
     hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, getHttpPort());
     hiveConf.setIntVar(ConfVars.HIVE_SERVER2_WEBUI_PORT, getWebPort());
+    hiveConf.setIntVar(ConfVars.HIVE_SERVER2_HEALTH_HA_PORT, getHealthHAPort());
 
     Path scratchDir = new Path(baseFsDir, "scratch");
     // Create root scratchdir with write all, so that user impersonation has no issues.