graceful shutdown for process-agent

cmd/agent/collector.go

@@ -5,12 +5,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/StackVista/stackstate-agent/pkg/aggregator"
-	"github.com/StackVista/stackstate-agent/pkg/batcher"
-	"github.com/StackVista/stackstate-agent/pkg/collector/check"
-	"github.com/StackVista/stackstate-agent/pkg/telemetry"
-	"github.com/StackVista/stackstate-agent/pkg/topology"
-	"github.com/StackVista/stackstate-process-agent/cmd/agent/features"
 	"io"
 	"io/ioutil"
 	"math/rand"
@@ -19,6 +13,13 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/StackVista/stackstate-agent/pkg/aggregator"
+	"github.com/StackVista/stackstate-agent/pkg/batcher"
+	"github.com/StackVista/stackstate-agent/pkg/collector/check"
+	"github.com/StackVista/stackstate-agent/pkg/telemetry"
+	"github.com/StackVista/stackstate-agent/pkg/topology"
+	"github.com/StackVista/stackstate-process-agent/cmd/agent/features"
+
 	log "github.com/cihub/seelog"
 
 	"github.com/StackVista/stackstate-process-agent/checks"
@@ -140,7 +141,7 @@ func (l *Collector) run(exit chan bool) {
 	}
 	log.Infof("Starting process-agent for host=%s, endpoints=%s, enabled checks=%v", l.cfg.HostName, eps, l.cfg.EnabledChecks)
 
-	go handleSignals(exit)
+	go HandleSignals(exit)
 	heartbeat := time.NewTicker(15 * time.Second)
 	queueSizeTicker := time.NewTicker(10 * time.Second)
 	featuresTicker := time.NewTicker(5 * time.Second)

cmd/agent/main_docker.go

@@ -12,21 +12,23 @@ import (
 	log "github.com/cihub/seelog"
 )
 
-// Handles signals - tells us whether we should exit.
-func handleSignals(exit chan bool) {
+// HandleSignals tells us whether we should exit.
+func HandleSignals(exit chan bool) {
 	sigIn := make(chan os.Signal, 100)
-	signal.Notify(sigIn)
-	// unix only in all likelihood;  but we don't care.
+	signal.Notify(sigIn, syscall.SIGINT, syscall.SIGTERM, syscall.SIGPIPE)
+	// unix only in all likelihood; but we don't care.
 	for sig := range sigIn {
 		switch sig {
 		case syscall.SIGINT, syscall.SIGTERM:
-			log.Criticalf("Caught signal '%s'; terminating.", sig)
-			close(exit)
-		case syscall.SIGCHLD:
-			// Running docker.GetDockerStat() spins up / kills a new process
+			log.Infof("Caught signal '%s'; terminating.", sig)
+			exit <- true
+			return
+		case syscall.SIGPIPE:
+			// By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal.
+			// Go ignores SIGPIPE signals unless it is when stderr or stdout is closed, in this case the agent is stopped.
+			// We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them.
+			// See https://golang.org/pkg/os/signal/#hdr-SIGPIPE
 			continue
-		default:
-			log.Warnf("Caught signal %s; continuing/ignoring.", sig)
 		}
 	}
 }

cmd/agent/main_nodocker.go

@@ -11,18 +11,23 @@ import (
 	log "github.com/cihub/seelog"
 )
 
-// Handles signals - tells us whether we should exit.
-func handleSignals(exit chan bool) {
+// HandleSignals tells us whether we should exit.
+func HandleSignals(exit chan bool) {
 	sigIn := make(chan os.Signal, 100)
-	signal.Notify(sigIn)
-	// unix only in all likelihood;  but we don't care.
+	signal.Notify(sigIn, syscall.SIGINT, syscall.SIGTERM, syscall.SIGPIPE)
+	// unix only in all likelihood; but we don't care.
 	for sig := range sigIn {
 		switch sig {
 		case syscall.SIGINT, syscall.SIGTERM:
-			log.Criticalf("Caught signal '%s'; terminating.", sig)
-			close(exit)
-		default:
-			log.Warnf("Caught signal %s; continuing/ignoring.", sig)
+			log.Infof("Caught signal '%s'; terminating.", sig)
+			exit <- true
+			return
+		case syscall.SIGPIPE:
+			// By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal.
+			// Go ignores SIGPIPE signals unless it is when stderr or stdout is closed, in this case the agent is stopped.
+			// We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them.
+			// See https://golang.org/pkg/os/signal/#hdr-SIGPIPE
+			continue
 		}
 	}
 }

cmd/network-tracer/main.go

@@ -92,9 +92,9 @@ func main() {
 	log.Infof("network tracer started")
 
 	// Handles signals, which tells us whether we should exit.
-	e := make(chan bool)
-	go handleSignals(e)
-	<-e
+	exit := make(chan bool)
+	go HandleSignals(exit)
+	<-exit
 }
 
 func gracefulExit() {
@@ -105,17 +105,23 @@ func gracefulExit() {
 	os.Exit(0)
 }
 
-func handleSignals(exit chan bool) {
+// HandleSignals tells us whether we should exit.
+func HandleSignals(exit chan bool) {
 	sigIn := make(chan os.Signal, 100)
-	signal.Notify(sigIn)
-	// unix only in all likelihood;  but we don't care.
+	signal.Notify(sigIn, syscall.SIGINT, syscall.SIGTERM, syscall.SIGPIPE)
+	// unix only in all likelihood; but we don't care.
 	for sig := range sigIn {
 		switch sig {
-		case syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGQUIT:
-			log.Criticalf("Caught signal '%s'; terminating.", sig)
-			close(exit)
-		default:
-			log.Warnf("Caught signal %s; continuing/ignoring.", sig)
+		case syscall.SIGINT, syscall.SIGTERM:
+			log.Infof("Caught signal '%s'; terminating.", sig)
+			exit <- true
+			return
+		case syscall.SIGPIPE:
+			// By default systemd redirects the stdout to journald. When journald is stopped or crashes we receive a SIGPIPE signal.
+			// Go ignores SIGPIPE signals unless it is when stderr or stdout is closed, in this case the agent is stopped.
+			// We never want the agent to stop upon receiving SIGPIPE, so we intercept the SIGPIPE signals and just discard them.
+			// See https://golang.org/pkg/os/signal/#hdr-SIGPIPE
+			continue
 		}
 	}
 }