Migrate CHGM investigation to use executor actions

This commit migrates the Cluster Has Gone Missing (CHGM) investigation to
use the new executor action pattern while maintaining proper error handling
for infrastructure failures.

**Key changes:**

1. **Action-based remediation:**
   - Replaced direct OCM/PagerDuty calls with declarative actions
   - Service logs, limited support, notes, and incident management now use
     executor action builders
   - Investigation returns actions to be executed by the executor

2. **Smart error handling:**
   - Added isInfrastructureError() helper to distinguish between:
     * Infrastructure failures (AWS/OCM API errors) → return error for retry
     * Investigation findings (data too old, inconclusive) → return actions
   - Resource building errors continue to return errors (no change)

3. **Removed dependencies:**
   - Deleted postStoppedInfraLimitedSupport() helper function
   - Removed utils.WithRetries() usage
   - Removed unused pagerduty package import

4. **Investigation outcomes now use actions:**
   - Stopped instances → Limited support + silence
   - Egress blocked (deadman's snitch) → Limited support + silence
   - Egress blocked (other URLs) → Service log + escalate
   - No issues found → Note + escalate
   - Investigation incomplete → Note + escalate

**Benefits:**
- Separation of investigation logic from external system updates
- Automatic retry for transient failures via executor
- Type-safe action builders (no interface{} types)
- Declarative results easier to test and understand
- Consistent error handling across investigations

**Breaking change:**
Tests need to be updated to verify actions instead of mocking direct
OCM/PagerDuty calls. This will be addressed in a follow-up commit.

Related: Phase 1 of executor module implementation

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

Author: bergmannf

pkg/investigations/chgm/chgm.go

@@ -8,13 +8,13 @@ import (
 	"time"
 
 	"github.com/openshift/configuration-anomaly-detection/pkg/aws"
+	"github.com/openshift/configuration-anomaly-detection/pkg/executor"
 	investigation "github.com/openshift/configuration-anomaly-detection/pkg/investigations/investigation"
+	"github.com/openshift/configuration-anomaly-detection/pkg/investigations/types"
 	"github.com/openshift/configuration-anomaly-detection/pkg/logging"
 	"github.com/openshift/configuration-anomaly-detection/pkg/networkverifier"
 	"github.com/openshift/configuration-anomaly-detection/pkg/notewriter"
 	"github.com/openshift/configuration-anomaly-detection/pkg/ocm"
-	"github.com/openshift/configuration-anomaly-detection/pkg/pagerduty"
-	"github.com/openshift/configuration-anomaly-detection/pkg/utils"
 	hivev1 "github.com/openshift/hive/apis/hive/v1"
 
 	ec2v2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
@@ -35,6 +35,52 @@ var (
 
 type Investiation struct{}
 
+// isInfrastructureError determines if an error is a transient infrastructure
+// failure that should cause the investigation to be retried (return error),
+// vs an investigation finding that should be reported (return actions).
+func isInfrastructureError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errStr := err.Error()
+
+	// Transient API failures that should trigger retry
+	infraPatterns := []string{
+		"could not retrieve non running instances",
+		"could not retrieve running cluster nodes",
+		"could not retrieve expected cluster nodes",
+		"could not PollStopEventsFor",
+		"failed to retrieve",
+		"timeout",
+		"rate limit",
+		"connection refused",
+		"service unavailable",
+	}
+
+	for _, pattern := range infraPatterns {
+		if strings.Contains(errStr, pattern) {
+			return true
+		}
+	}
+
+	// Investigation findings that should be reported (NOT retried)
+	findingPatterns := []string{
+		"stopped instances but no stoppedInstancesEvents", // CloudTrail data too old
+		"clusterdeployment is empty",                      // Data missing but not retriable
+		"no non running instances found",                  // Investigation finding
+	}
+
+	for _, pattern := range findingPatterns {
+		if strings.Contains(errStr, pattern) {
+			return false
+		}
+	}
+
+	// Default: assume infrastructure error to be safe (retry)
+	return true
+}
+
 // Run runs the investigation for a triggered chgm pagerduty event
 func (c *Investiation) Run(rb investigation.ResourceBuilder) (investigation.InvestigationResult, error) {
 	result := investigation.InvestigationResult{}
@@ -48,19 +94,34 @@ func (c *Investiation) Run(rb investigation.ResourceBuilder) (investigation.Inve
 	// 1. Check if the user stopped instances
 	res, err := investigateStoppedInstances(r.Cluster, r.ClusterDeployment, r.AwsClient, r.OcmClient)
 	if err != nil {
-		return result, r.PdClient.EscalateIncidentWithNote(fmt.Sprintf("InvestigateInstances failed: %s\n", err.Error()))
+		// Check if this is a transient infrastructure error (AWS/OCM API failures)
+		// These should trigger a retry of the entire investigation
+		if isInfrastructureError(err) {
+			return result, fmt.Errorf("investigation infrastructure failure: %w", err)
+		}
+
+		// Otherwise, it's an investigation finding (e.g., CloudTrail data too old)
+		// Report this as a finding that needs manual investigation
+		notes.AppendWarning("Could not complete instance investigation: %s", err.Error())
+		result.Actions = []types.Action{
+			executor.NoteFrom(notes),
+			executor.Escalate("Investigation incomplete - manual review required"),
+		}
+		return result, nil
 	}
 	logging.Debugf("the investigation returned: [infras running: %d] - [masters running: %d]", res.RunningInstances.Infra, res.RunningInstances.Master)
 
 	if !res.UserAuthorized {
 		logging.Infof("Instances were stopped by unauthorized user: %s / arn: %s", res.User.UserName, res.User.IssuerUserName)
-		return result, utils.WithRetries(func() error {
-			err := postStoppedInfraLimitedSupport(r.Cluster, r.OcmClient, r.PdClient)
-			// XXX: metrics.Inc(metrics.ServicelogSent, investigationName)
-			result.LimitedSupportSet = investigation.InvestigationStep{Performed: true, Labels: []string{"StoppedInstances"}}
+		notes.AppendAutomation("Customer stopped instances. Sent LS and silencing alert.")
 
-			return err
-		})
+		result.LimitedSupportSet = investigation.InvestigationStep{Performed: true, Labels: []string{"StoppedInstances"}}
+		result.Actions = []types.Action{
+			executor.NewLimitedSupportAction(stoppedInfraLS.Summary, stoppedInfraLS.Details).Build(),
+			executor.NoteFrom(notes),
+			executor.Silence("Customer stopped instances - cluster in limited support"),
+		}
+		return result, nil
 	}
 	notes.AppendSuccess("Customer did not stop nodes.")
 	logging.Info("The customer has not stopped/terminated any nodes.")
@@ -92,36 +153,45 @@ func (c *Investiation) Run(rb investigation.ResourceBuilder) (investigation.Inve
 		logging.Infof("Network verifier reported failure: %s", failureReason)
 
 		if strings.Contains(failureReason, "nosnch.in") {
-			err := r.OcmClient.PostLimitedSupportReason(r.Cluster, &egressLS)
-			if err != nil {
-				return result, err
-			}
+			notes.AppendAutomation("Egress `nosnch.in` blocked, sent limited support.")
 
-			// XXX: metrics.Inc(metrics.LimitedSupportSet, investigationName, "EgressBlocked")
 			result.LimitedSupportSet = investigation.InvestigationStep{Performed: true, Labels: []string{"EgressBlocked"}}
-
-			notes.AppendAutomation("Egress `nosnch.in` blocked, sent limited support.")
-			return result, r.PdClient.SilenceIncidentWithNote(notes.String())
+			result.Actions = []types.Action{
+				executor.NewLimitedSupportAction(egressLS.Summary, egressLS.Details).
+					WithContext("EgressBlocked").
+					Build(),
+				executor.NoteFrom(notes),
+				executor.Silence("Deadman's snitch blocked - cluster in limited support"),
+			}
+			return result, nil
 		}
 
 		docLink := ocm.DocumentationLink(product, ocm.DocumentationTopicPrivatelinkFirewall)
 		egressSL := createEgressSL(failureReason, docLink)
-		err := r.OcmClient.PostServiceLog(r.Cluster, egressSL)
-		if err != nil {
-			return result, err
-		}
-
-		// XXX: metrics.Inc(metrics.ServicelogSent, investigationName)
-		result.ServiceLogSent = investigation.InvestigationStep{Performed: true, Labels: nil}
 
 		notes.AppendWarning("NetworkVerifier found unreachable targets and sent the SL, but deadmanssnitch is not blocked! \n⚠️ Please investigate this cluster.\nUnreachable: \n%s", failureReason)
+
+		result.ServiceLogSent = investigation.InvestigationStep{Performed: true, Labels: nil}
+		result.Actions = []types.Action{
+			executor.NewServiceLogAction(egressSL.Severity, egressSL.Summary).
+				WithDescription(egressSL.Description).
+				WithServiceName(egressSL.ServiceName).
+				Build(),
+			executor.NoteFrom(notes),
+			executor.Escalate("Egress blocked but not deadman's snitch - manual investigation required"),
+		}
+		return result, nil
 	case networkverifier.Success:
 		notes.AppendSuccess("Network verifier passed")
 		logging.Info("Network verifier passed.")
 	}
 
 	// Found no issues that CAD can handle by itself - forward notes to SRE.
-	return result, r.PdClient.EscalateIncidentWithNote(notes.String())
+	result.Actions = []types.Action{
+		executor.NoteFrom(notes),
+		executor.Escalate("No automated remediation available - manual investigation required"),
+	}
+	return result, nil
 }
 
 func (c *Investiation) Name() string {
@@ -453,12 +523,4 @@ func extractUserDetails(cloudTrailEvent *string) (CloudTrailEventRaw, error) {
 	return res, nil
 }
 
-// postStoppedInfraLimitedSupport will put the cluster on limited support because the user has stopped instances
-func postStoppedInfraLimitedSupport(cluster *cmv1.Cluster, ocmCli ocm.Client, pdCli pagerduty.Client) error {
-	err := ocmCli.PostLimitedSupportReason(cluster, &stoppedInfraLS)
-	if err != nil {
-		return fmt.Errorf("failed sending limited support reason: %w", err)
-	}
 
-	return pdCli.SilenceIncidentWithNote("Customer stopped instances. Sent SL and silencing alert.")
-}