HIVE-28775: HiveServer2: enabling HA Leader endpoint on a different port than WebUI

Author: Dmitriy Fingerman

common/src/java/org/apache/hadoop/hive/conf/HiveConf.java

@@ -3979,6 +3979,8 @@ public static enum ConfVars {
     HIVE_SERVER2_WEBUI_BIND_HOST("hive.server2.webui.host", "0.0.0.0", "The host address the HiveServer2 WebUI will listen on"),
     HIVE_SERVER2_WEBUI_PORT("hive.server2.webui.port", 10002, "The port the HiveServer2 WebUI will listen on. This can be"
         + "set to 0 or a negative integer to disable the web UI"),
+    HIVE_SERVER2_HEALTH_HA_PORT("hive.server2.health.ha.port", 11002, "The port the HiveServer2 health-ha web app will listen on. This can be"
+        + "set to 0 or a negative integer to disable HS2 health-ha checking"),
     HIVE_SERVER2_WEBUI_MAX_THREADS("hive.server2.webui.max.threads", 50, "The max HiveServer2 WebUI threads"),
     HIVE_SERVER2_WEBUI_USE_SSL("hive.server2.webui.use.ssl", false,
         "Set this to true for using SSL encryption for HiveServer2 WebUI."),

common/src/java/org/apache/hive/http/HttpServer.java

@@ -82,15 +82,17 @@
 import org.eclipse.jetty.security.ConstraintMapping;
 import org.eclipse.jetty.security.ConstraintSecurityHandler;
 import org.eclipse.jetty.security.LoginService;
-import org.eclipse.jetty.server.Connector;
 import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.LowResourceMonitor;
+import org.eclipse.jetty.server.Request;
 import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.handler.AbstractHandler;
 import org.eclipse.jetty.server.handler.ContextHandler.Context;
 import org.eclipse.jetty.server.handler.ContextHandlerCollection;
 import org.eclipse.jetty.server.ServerConnector;
+import org.eclipse.jetty.server.handler.HandlerCollection;
 import org.eclipse.jetty.servlet.DefaultServlet;
 import org.eclipse.jetty.servlet.FilterHolder;
 import org.eclipse.jetty.servlet.FilterMapping;
@@ -138,6 +140,8 @@ public class HttpServer {
   private String appDir;
   private WebAppContext webAppContext;
   private Server webServer;
+  private QueuedThreadPool threadPool;
+  private PortHandlerWrapper portHandlerWrapper;
 
   /**
    * Create a status server on the given port.
@@ -179,6 +183,7 @@ public static class Builder {
         new LinkedList<Pair<String, Class<? extends HttpServlet>>>();
     private boolean disableDirListing = false;
     private final Map<String, Pair<String, Filter>> globalFilters = new LinkedHashMap<>();
+    private String contextPath;
 
     public Builder(String name) {
       Preconditions.checkArgument(name != null && !name.isEmpty(), "Name must be specified");
@@ -189,6 +194,10 @@ public HttpServer build() throws IOException {
       return new HttpServer(this);
     }
 
+    public void setContextPath(String contextPath) {
+      this.contextPath = contextPath;
+    }
+
     public Builder setConf(HiveConf origConf) {
       this.conf = new HiveConf(origConf);
       origConf.stripHiddenConfigurations(conf);
@@ -489,13 +498,13 @@ static boolean userHasAdministratorAccess(ServletContext servletContext,
   /**
    * Create the web context for the application of specified name
    */
-  WebAppContext createWebAppContext(Builder b) {
+  WebAppContext createWebAppContext(Builder b) throws FileNotFoundException {
     WebAppContext ctx = new WebAppContext();
     setContextAttributes(ctx.getServletContext(), b.contextAttrs);
     ctx.getServletContext().getSessionCookieConfig().setHttpOnly(true);
     ctx.setDisplayName(b.name);
-    ctx.setContextPath("/");
-    ctx.setWar(appDir + "/" + b.name);
+    ctx.setContextPath(b.contextPath == null ? "/" : b.contextPath);
+    ctx.setWar(getWebAppsPath(b.name) + "/" + b.name);
     return ctx;
   }
 
@@ -519,8 +528,9 @@ void setupSpnegoFilter(Builder b, ServletContextHandler ctx) throws IOException
   /**
    * Setup cross-origin requests (CORS) filter.
    * @param b - builder
+   * @param webAppContext - webAppContext
    */
-  private void setupCORSFilter(Builder b) {
+  private void setupCORSFilter(Builder b, WebAppContext webAppContext) {
     FilterHolder holder = new FilterHolder();
     holder.setClassName(CrossOriginFilter.class.getName());
     Map<String, String> params = new HashMap<>();
@@ -533,10 +543,62 @@ private void setupCORSFilter(Builder b) {
     handler.addFilterWithMapping(holder, "/*", FilterMapping.ALL);
   }
 
+  public void addWebApp(Builder b) throws IOException {
+    WebAppContext webAppContext = createWebAppContext(b);
+    ContextHandlerCollection portHandler = new ContextHandlerCollection();
+    ServerConnector connector = addWebApp(b, webAppContext, portHandler);
+    portHandlerWrapper.addHandler(connector, portHandler);
+  }
+  
+  private ServerConnector addWebApp(Builder b, WebAppContext webAppContext, ContextHandlerCollection portHandler)
+      throws IOException {
+
+    if (b.useSPNEGO) {
+      // Secure the web server with kerberos
+      setupSpnegoFilter(b, webAppContext);
+    }
+
+    if (b.enableCORS) {
+      setupCORSFilter(b, webAppContext);
+    }
+
+    Map<String, String> xFrameParams = setHeaders();
+    if (b.xFrameEnabled) {
+      setupXframeFilter(b, xFrameParams, webAppContext);
+    }
+
+    if (b.disableDirListing) {
+      disableDirectoryListingOnServlet(webAppContext);
+    }
+
+    ServerConnector connector = createChannelConnector(threadPool.getQueueSize(), b);
+    webServer.addConnector(connector);
+
+    RewriteHandler rwHandler = new RewriteHandler();
+    rwHandler.setRewriteRequestURI(true);
+    rwHandler.setRewritePathInfo(false);
+
+    RewriteRegexRule rootRule = new RewriteRegexRule();
+    rootRule.setRegex("^/$");
+    rootRule.setReplacement(b.contextRootRewriteTarget);
+    rootRule.setTerminating(true);
+
+    rwHandler.addRule(rootRule);
+    rwHandler.setHandler(webAppContext);
+
+    portHandler.addHandler(rwHandler);
+
+    for (Pair<String, Class<? extends HttpServlet>> p : b.servlets) {
+      addServlet(p.getFirst(), "/" + p.getFirst(), p.getSecond(), webAppContext);
+    }
+    
+    return connector;
+  }
+  
   /**
    * Create a channel connector for "http/https" requests
    */
-  Connector createChannelConnector(int queueSize, Builder b) {
+  ServerConnector createChannelConnector(int queueSize, Builder b) {
     ServerConnector connector;
 
     final HttpConfiguration conf = new HttpConfiguration();
@@ -604,7 +666,7 @@ void setContextAttributes(Context ctx, Map<String, Object> contextAttrs) {
 
   private void createWebServer(final Builder b) throws IOException {
     // Create the thread pool for the web server to handle HTTP requests
-    QueuedThreadPool threadPool = new QueuedThreadPool();
+    threadPool = new QueuedThreadPool();
     if (b.maxThreads > 0) {
       threadPool.setMaxThreads(b.maxThreads);
     }
@@ -615,59 +677,29 @@ private void createWebServer(final Builder b) throws IOException {
     this.appDir = getWebAppsPath(b.name);
     this.webAppContext = createWebAppContext(b);
 
-    if (b.useSPNEGO) {
-      // Secure the web server with kerberos
-      setupSpnegoFilter(b, webAppContext);
-    }
-
-    if (b.enableCORS) {
-      setupCORSFilter(b);
-    }
-
-    Map<String, String> xFrameParams = setHeaders();
-    if (b.xFrameEnabled) {
-      setupXframeFilter(b,xFrameParams);
-    }
-
-    if (b.disableDirListing) {
-      disableDirectoryListingOnServlet(webAppContext);
-    }
-
-    initializeWebServer(b, threadPool.getMaxThreads());
+    initializeWebServer(b);
   }
 
-  private void initializeWebServer(final Builder b, int queueSize) throws IOException {
+  private void initializeWebServer(final Builder b) throws IOException {
     // Set handling for low resource conditions.
     final LowResourceMonitor low = new LowResourceMonitor(webServer);
     low.setLowResourcesIdleTimeout(10000);
     webServer.addBean(low);
 
-    Connector connector = createChannelConnector(queueSize, b);
-    webServer.addConnector(connector);
-
-    RewriteHandler rwHandler = new RewriteHandler();
-    rwHandler.setRewriteRequestURI(true);
-    rwHandler.setRewritePathInfo(false);
-
-    RewriteRegexRule rootRule = new RewriteRegexRule();
-    rootRule.setRegex("^/$");
-    rootRule.setReplacement(b.contextRootRewriteTarget);
-    rootRule.setTerminating(true);
-
-    rwHandler.addRule(rootRule);
-    rwHandler.setHandler(webAppContext);
-
-    // Configure web application contexts for the web server
-    ContextHandlerCollection contexts = new ContextHandlerCollection();
-    contexts.addHandler(rwHandler);
-    webServer.setHandler(contexts);
-
+    // Configure the global context handler collection for the web server
+    portHandlerWrapper = new PortHandlerWrapper();
+    webServer.setHandler(portHandlerWrapper);
+    
+    ContextHandlerCollection portHandler = new ContextHandlerCollection();
+    portHandler.addHandler(webAppContext);
+    ServerConnector connector = addWebApp(b, webAppContext, portHandler);
+    
+    portHandlerWrapper.addHandler(connector, portHandler);
 
     if (b.usePAM) {
-      setupPam(b, contexts);
+      setupPam(b, portHandlerWrapper);
     }
 
-
     addServlet("jmx", "/jmx", JMXJsonServlet.class);
     addServlet("conf", "/conf", ConfServlet.class);
     addServlet("stacks", "/stacks", StackServlet.class);
@@ -679,8 +711,7 @@ private void initializeWebServer(final Builder b, int queueSize) throws IOExcept
       if (Files.notExists(tmpDir)) {
         Files.createDirectories(tmpDir);
       }
-      ServletContextHandler genCtx =
-        new ServletContextHandler(contexts, "/prof-output");
+      ServletContextHandler genCtx = new ServletContextHandler(portHandler, "/prof-output");
       setContextAttributes(genCtx.getServletContext(), b.contextAttrs);
       genCtx.addServlet(ProfileOutputServlet.class, "/*");
       genCtx.setResourceBase(tmpDir.toAbsolutePath().toString());
@@ -689,23 +720,17 @@ private void initializeWebServer(final Builder b, int queueSize) throws IOExcept
       LOG.info("ASYNC_PROFILER_HOME env or -Dasync.profiler.home not specified. Disabling /prof endpoint..");
     }
 
-    for (Pair<String, Class<? extends HttpServlet>> p : b.servlets) {
-      addServlet(p.getFirst(), "/" + p.getFirst(), p.getSecond());
-    }
-
     b.globalFilters.forEach((k, v) -> addFilter(k, v.getFirst(), v.getSecond(), webAppContext.getServletHandler()));
 
-    ServletContextHandler staticCtx =
-      new ServletContextHandler(contexts, "/static");
+    ServletContextHandler staticCtx = new ServletContextHandler(portHandler, "/static");
     staticCtx.setResourceBase(appDir + "/static");
     staticCtx.addServlet(DefaultServlet.class, "/*");
     staticCtx.setDisplayName("static");
     disableDirectoryListingOnServlet(staticCtx);
 
     String logDir = getLogDir(b.conf);
     if (logDir != null) {
-      ServletContextHandler logCtx =
-        new ServletContextHandler(contexts, "/logs");
+      ServletContextHandler logCtx = new ServletContextHandler(portHandler, "/logs");
       setContextAttributes(logCtx.getServletContext(), b.contextAttrs);
       if(b.useSPNEGO) {
         setupSpnegoFilter(b,logCtx);
@@ -716,7 +741,7 @@ private void initializeWebServer(final Builder b, int queueSize) throws IOExcept
     }
 
     // Define the global filers for each servlet context except the staticCtx(css style).
-    Optional<Handler[]> handlers = Optional.ofNullable(contexts.getHandlers());
+    Optional<Handler[]> handlers = Optional.ofNullable(portHandler.getHandlers());
     handlers.ifPresent(hs -> Arrays.stream(hs)
         .filter(h -> h instanceof ServletContextHandler && !"static".equals(((ServletContextHandler) h).getDisplayName()))
         .forEach(h -> b.globalFilters.forEach((k, v) ->
@@ -748,7 +773,7 @@ private Map<String, String> getDefaultHeaders() {
     return headers;
   }
 
-  private void setupXframeFilter(Builder b, Map<String, String> params) {
+  private void setupXframeFilter(Builder b, Map<String, String> params, WebAppContext webAppContext) {
     FilterHolder holder = new FilterHolder();
     holder.setClassName(QuotingInputFilter.class.getName());
     holder.setInitParameters(params);
@@ -811,6 +836,15 @@ public void addServlet(String name, String pathSpec,
     webAppContext.addServlet(holder, pathSpec);
   }
 
+  private void addServlet(String name, String pathSpec, Class<? extends HttpServlet> clazz,
+      WebAppContext webAppContext) {
+    ServletHolder holder = new ServletHolder(clazz);
+    if (name != null) {
+      holder.setName(name);
+    }
+    webAppContext.addServlet(holder, pathSpec);
+  }
+
   public void addServlet(String name, String pathSpec, ServletHolder holder) {
     if (name != null) {
       holder.setName(name);
@@ -1026,4 +1060,32 @@ private void initHttpHeaderMap() {
     }
   }
 
+  class PortHandlerWrapper extends ContextHandlerCollection {
+
+    private final Map<ServerConnector, HandlerCollection> connectorToHandlerMap = new HashMap<>();
+
+    public PortHandlerWrapper() {
+    }
+    
+    public void addHandler(ServerConnector connector, HandlerCollection handler) {
+      connectorToHandlerMap.put(connector, handler);
+      addHandler(handler);
+    }
+
+    @Override
+    public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
+      // Determine the connector (port) the request came through
+      int port = request.getServerPort();
+
+      // Find the handler for the corresponding port
+      Handler handler = connectorToHandlerMap.entrySet().stream()
+          .filter(entry -> entry.getKey().getPort() == port)
+          .map(Map.Entry::getValue)
+          .findFirst()
+          .orElseThrow(() -> new IllegalStateException("No handler found for port " + port));
+
+      // Delegate the request to the handler
+      handler.handle(target, baseRequest, request, response);
+    }
+  }
 }

data/conf/hive-site.xml

@@ -358,7 +358,7 @@
 
   <property>
     <name>hive.server2.webui.max.threads</name>
-    <value>4</value>
+    <value>6</value>
   </property>
 
   <property>

data/conf/mr/hive-site.xml

@@ -354,7 +354,7 @@
 
   <property>
     <name>hive.server2.webui.max.threads</name>
-    <value>4</value>
+    <value>6</value>
   </property>
 
   <property>