Merge pull request #4196 from pmtk/recover-and-repanic

USHIFT-4519: Recover and re-panic certain subcomponents
openshift · Nov 21, 2024 · 4de142b · 4de142b
2 parents 8af8fd0 + 0f9692a
commit 4de142b
Show file tree

Hide file tree

Showing 6 changed files with 76 additions and 6 deletions.
diff --git a/pkg/controllers/kube-apiserver.go b/pkg/controllers/kube-apiserver.go
@@ -389,8 +389,21 @@ func (s *KubeAPIServer) Run(ctx context.Context, ready chan<- struct{}, stopped
 		"--openshift-config", fd.Name(),
 		"-v", strconv.Itoa(s.verbosity),
 	})
+
+	panicChannel := make(chan any, 1)
 	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				panicChannel <- r
+			}
+		}()
 		errorChannel <- cmd.ExecuteContext(ctx)
 	}()
-	return <-errorChannel
+
+	select {
+	case err := <-errorChannel:
+		return err
+	case perr := <-panicChannel:
+		panic(perr)
+	}
 }
diff --git a/pkg/controllers/kube-controller-manager.go b/pkg/controllers/kube-controller-manager.go
@@ -138,14 +138,27 @@ func (s *KubeControllerManager) Run(ctx context.Context, ready chan<- struct{},
 	// which expects to be called at most once in a process.
 	cmd := kubecm.NewControllerManagerCommand()
 	cmd.SetArgs(s.args)
+
+	panicChannel := make(chan any, 1)
 	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				panicChannel <- r
+			}
+		}()
 		errorChannel <- cmd.ExecuteContext(ctx)
 	}()
 
 	if err := s.applyFn(); err != nil {
 		return fmt.Errorf("failed to apply openshift namespaces: %w", err)
 	}
-	return <-errorChannel
+
+	select {
+	case err := <-errorChannel:
+		return err
+	case perr := <-panicChannel:
+		panic(perr)
+	}
 }
 
 func mergeAndConvertToArgs(overrides *kubecontrolplanev1.KubeControllerManagerConfig) ([]string, error) {

diff --git a/pkg/controllers/kube-scheduler.go b/pkg/controllers/kube-scheduler.go
@@ -96,9 +96,20 @@ func (s *KubeScheduler) Run(ctx context.Context, ready chan<- struct{}, stopped
 		return err
 	}
 
+	panicChannel := make(chan any, 1)
 	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				panicChannel <- r
+			}
+		}()
 		errorChannel <- kubescheduler.Run(ctx, cc, sched)
 	}()
 
-	return <-errorChannel
+	select {
+	case err := <-errorChannel:
+		return err
+	case perr := <-panicChannel:
+		panic(perr)
+	}
 }
diff --git a/pkg/controllers/kube-storage-version-migrator.go b/pkg/controllers/kube-storage-version-migrator.go
@@ -73,11 +73,22 @@ func (s *KubeStorageVersionMigrator) Run(ctx context.Context, ready chan<- struc
 		close(ready)
 	}()
 
+	panicChannel := make(chan any, 1)
 	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				panicChannel <- r
+			}
+		}()
 		errorChannel <- s.runMigrator(ctx)
 	}()
 
-	return <-errorChannel
+	select {
+	case err := <-errorChannel:
+		return err
+	case perr := <-panicChannel:
+		panic(perr)
+	}
 }
 
 func (s *KubeStorageVersionMigrator) runMigrator(ctx context.Context) error {

diff --git a/pkg/controllers/openshift-route-controller-manager.go b/pkg/controllers/openshift-route-controller-manager.go
@@ -187,9 +187,20 @@ func (s *OCPRouteControllerManager) Run(ctx context.Context, ready chan<- struct
 		close(ready)
 	}()
 
+	panicChannel := make(chan any, 1)
 	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				panicChannel <- r
+			}
+		}()
 		errc <- s.run(ctx)
 	}()
 
-	return <-errc
+	select {
+	case err := <-errc:
+		return err
+	case perr := <-panicChannel:
+		panic(perr)
+	}
 }
diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go
@@ -192,11 +192,22 @@ func (s *KubeletServer) Run(ctx context.Context, ready chan<- struct{}, stopped
 		close(ready)
 	}()
 
+	panicChannel := make(chan any, 1)
 	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				panicChannel <- r
+			}
+		}()
 		errc <- kubelet.Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate)
 	}()
 
-	return <-errc
+	select {
+	case err := <-errc:
+		return err
+	case perr := <-panicChannel:
+		panic(perr)
+	}
 }
 
 func loadConfigFile(name string) (*kubeletconfig.KubeletConfiguration, error) {