Merge pull request #18 from project-aethermesh/fix/http-server-startup

fix: Start HTTP server before dependencies to ensure immediate availability

Author: sanbotto

internal/health/server.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"net/http"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/rs/zerolog/log"
@@ -14,6 +15,7 @@ import (
 // HealthCheckerServer provides HTTP endpoints for health and readiness checks
 type HealthCheckerServer struct {
 	checker    *Checker
+	checkerMu  sync.RWMutex // Protects checker field from concurrent access
 	httpServer *http.Server
 }
 
@@ -39,10 +41,20 @@ func NewHealthCheckerServer(port int, checker *Checker) *HealthCheckerServer {
 	return server
 }
 
+// SetChecker updates the checker instance after the server has started
+// This allows the server to start before dependencies (config, Valkey) are available
+func (s *HealthCheckerServer) SetChecker(checker *Checker) {
+	s.checkerMu.Lock()
+	defer s.checkerMu.Unlock()
+	s.checker = checker
+	log.Info().Msg("Health checker HTTP server: checker instance updated")
+}
+
 // Start starts the HTTP server in a goroutine and reports bind errors via the error channel
 func (s *HealthCheckerServer) Start(startupErrCh chan<- error) {
 	log.Info().Str("addr", s.httpServer.Addr).Msg("Starting health checker HTTP server")
 	go func() {
+		log.Debug().Str("addr", s.httpServer.Addr).Msg("HTTP server goroutine started, attempting to bind")
 		// First attempt to bind to detect immediate errors (port in use, permission denied, etc.)
 		listener, err := net.Listen("tcp", s.httpServer.Addr)
 		if err != nil {
@@ -51,6 +63,7 @@ func (s *HealthCheckerServer) Start(startupErrCh chan<- error) {
 			return
 		}
 
+		log.Info().Str("addr", s.httpServer.Addr).Msg("Successfully bound to address, starting HTTP server")
 		// Successfully bound, send nil to indicate successful startup
 		startupErrCh <- nil
 
@@ -90,9 +103,13 @@ func (s *HealthCheckerServer) handleReady(w http.ResponseWriter, r *http.Request
 	var statusCode int
 
 	// Read ready state under lock, then unlock before I/O
+	s.checkerMu.RLock()
+	checker := s.checker
+	s.checkerMu.RUnlock()
+
 	var isReady bool
-	if s.checker != nil {
-		isReady = s.checker.IsReady()
+	if checker != nil {
+		isReady = checker.IsReady()
 	} else {
 		isReady = false
 	}

internal/server/server.go

@@ -353,7 +353,6 @@ func (s *Server) checkHealthCheckerServiceReady(ctx context.Context) bool {
 	if resp.StatusCode == http.StatusOK {
 		// Drain response body to enable connection reuse
 		io.Copy(io.Discard, resp.Body)
-		log.Debug().Str("url", checkURL).Msg("Health-checker service is ready")
 		return true
 	}
 

services/health-checker/main.go

@@ -33,6 +33,9 @@ var testCheckerPatch func(*health.Checker)
 // testExitAfterSetup is a test hook to exit main after setup in tests
 var testExitAfterSetup bool
 
+// exitCode is used to track the exit code for the process
+var exitCode int
+
 // createStandaloneRateLimitHandler creates a simple rate limit handler for the standalone health checker
 func createStandaloneRateLimitHandler(valkeyClient store.ValkeyClientIface) func(chain, endpointID, protocol string) {
 	return func(chain, endpointID, protocol string) {
@@ -100,6 +103,29 @@ func RunHealthChecker(
 		return
 	}
 
+	// Start HTTP server FIRST, before any dependencies (config, Valkey)
+	// This ensures health probes are able to be used from the start
+	log.Info().Int("port", healthCheckerServerPort).Msg("Starting HTTP server before dependencies")
+	httpServer := health.NewHealthCheckerServer(healthCheckerServerPort, nil) // Start with nil checker
+	startupErrCh := make(chan error, 1)
+	httpServer.Start(startupErrCh)
+
+	// Wait for startup result from the HTTP server goroutine (bind errors are immediate)
+	if err := <-startupErrCh; err != nil {
+		log.Error().Err(err).Msg("Health checker HTTP server failed to start")
+		exitCode = 1
+		return
+	}
+	log.Info().Msg("HTTP server startup successful, proceeding with dependency initialization")
+
+	defer func() {
+		shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer shutdownCancel()
+		if err := httpServer.Shutdown(shutdownCtx); err != nil {
+			log.Error().Err(err).Msg("Error shutting down health checker HTTP server")
+		}
+	}()
+
 	// Start the metrics server if enabled
 	if metricsEnabled {
 		log.Info().Int("port", metricsPort).Msg("Prometheus metrics server enabled")
@@ -108,7 +134,9 @@ func RunHealthChecker(
 
 	cfg, err := loadConfig(configFile)
 	if err != nil {
-		log.Fatal().Err(err).Msg("Failed to load configuration")
+		log.Error().Err(err).Msg("Failed to load configuration")
+		exitCode = 1
+		return
 	}
 
 	log.Info().Msg("Health Checker Service - Loaded configuration:")
@@ -130,7 +158,9 @@ func RunHealthChecker(
 	valkeyAddr := valkeyHost + ":" + valkeyPort
 	valkeyClient := newValkeyClient(valkeyAddr, valkeyPass, valkeySkipTLSCheck, valkeyUseTLS)
 	if err := valkeyClient.Ping(context.Background()); err != nil {
-		log.Fatal().Err(err).Msg("Failed to connect to Valkey")
+		log.Error().Err(err).Msg("Failed to connect to Valkey")
+		exitCode = 1
+		return
 	}
 	defer valkeyClient.Close()
 
@@ -145,29 +175,9 @@ func RunHealthChecker(
 		testCheckerPatch(checker)
 	}
 
-	// Start HTTP server for health and readiness endpoints
-	httpServer := health.NewHealthCheckerServer(healthCheckerServerPort, checker)
-	startupErrCh := make(chan error, 1)
-	httpServer.Start(startupErrCh)
-
-	// Wait briefly to detect startup failures (bind errors should be immediate)
-	select {
-	case err := <-startupErrCh:
-		if err != nil {
-			log.Fatal().Err(err).Msg("Health checker HTTP server failed to start")
-		}
-		// Startup successful
-	case <-time.After(100 * time.Millisecond):
-		// No error received within timeout, assume startup successful
-		// Bind errors from net.Listen() should be immediate, so this is safe
-	}
-	defer func() {
-		shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer shutdownCancel()
-		if err := httpServer.Shutdown(shutdownCtx); err != nil {
-			log.Error().Err(err).Msg("Error shutting down health checker HTTP server")
-		}
-	}()
+	// Update HTTP server with checker instance now that dependencies are loaded
+	log.Info().Msg("Dependencies loaded, updating HTTP server with checker instance")
+	httpServer.SetChecker(checker)
 
 	if healthCheckInterval == 0 {
 		mode = "ephemeral"
@@ -241,4 +251,8 @@ func main() {
 		config.ValkeyUseTLS,
 		config.StandaloneHealthChecks,
 	)
+
+	// Exit with the appropriate code after RunHealthChecker returns
+	// This allows defers in RunHealthChecker to run, while not hiding panics
+	os.Exit(exitCode)
 }