pkg/start: Fill in deferred HandleCrash

Clayton wants these in each goroutine we launch [1].  Obviously
there's no way to reach inside the informer Start()s and add it there.
I'm also adding this to the FIXME comment for rerolling the
auto-update worker goroutines; we'll get those straigtened out in
future work.

[1]: openshift#424

Author: wking

pkg/autoupdate/autoupdate.go

@@ -85,7 +85,6 @@ func New(
 
 // Run runs the autoupdate controller.
 func (ctrl *Controller) Run(ctx context.Context, workers int) error {
-	defer utilruntime.HandleCrash()
 	defer ctrl.queue.ShutDown()
 
 	klog.Info("Starting AutoUpdateController")
@@ -96,7 +95,7 @@ func (ctrl *Controller) Run(ctx context.Context, workers int) error {
 	}
 
 	for i := 0; i < workers; i++ {
-		// FIXME: actually wait until these complete if the Context is canceled.
+		// FIXME: actually wait until these complete if the Context is canceled.  And possibly add utilruntime.HandleCrash.
 		go wait.UntilWithContext(ctx, ctrl.worker, time.Second)
 	}
 

pkg/cvo/cvo.go

@@ -291,7 +291,6 @@ func loadConfigMapVerifierDataFromUpdate(update *payload.Update, clientBuilder s
 
 // Run runs the cluster version operator until stopCh is completed. Workers is ignored for now.
 func (optr *Operator) Run(ctx context.Context, workers int) error {
-	defer utilruntime.HandleCrash()
 	defer optr.queue.ShutDown()
 	stopCh := ctx.Done()
 	workerStopCh := make(chan struct{})

pkg/start/start.go

@@ -16,6 +16,7 @@ import (
 	"github.com/google/uuid"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -184,6 +185,7 @@ func (o *Options) run(ctx context.Context, controllerCtx *Context, lock *resourc
 	defer func() { signal.Stop(ch) }()
 	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
 	go func() {
+		defer utilruntime.HandleCrash()
 		sig := <-ch
 		klog.Infof("Shutting down due to %s", sig)
 		runCancel()
@@ -204,6 +206,7 @@ func (o *Options) run(ctx context.Context, controllerCtx *Context, lock *resourc
 		}
 		resultChannelCount++
 		go func() {
+			defer utilruntime.HandleCrash()
 			err := cvo.RunMetrics(postMainContext, shutdownContext, o.ListenAddr, tlsConfig)
 			resultChannel <- asyncResult{name: "metrics server", error: err}
 		}()
@@ -218,6 +221,7 @@ func (o *Options) run(ctx context.Context, controllerCtx *Context, lock *resourc
 
 	resultChannelCount++
 	go func() {
+		defer utilruntime.HandleCrash()
 		leaderelection.RunOrDie(postMainContext, leaderelection.LeaderElectionConfig{
 			Lock:            lock,
 			ReleaseOnCancel: true,
@@ -228,13 +232,15 @@ func (o *Options) run(ctx context.Context, controllerCtx *Context, lock *resourc
 				OnStartedLeading: func(_ context.Context) { // no need for this passed-through postMainContext, because goroutines we launch inside will use runContext
 					resultChannelCount++
 					go func() {
+						defer utilruntime.HandleCrash()
 						err := controllerCtx.CVO.Run(runContext, 2)
 						resultChannel <- asyncResult{name: "main operator", error: err}
 					}()
 
 					if controllerCtx.AutoUpdate != nil {
 						resultChannelCount++
 						go func() {
+							defer utilruntime.HandleCrash()
 							err := controllerCtx.AutoUpdate.Run(runContext, 2)
 							resultChannel <- asyncResult{name: "auto-update controller", error: err}
 						}()