Add stale branches scannning support (#285)

* Add stale branches scanning support
Signed-off-by: Sébastien Graveline <[email protected]>
Co-authored-by: Copilot <[email protected]>
Co-authored-by: François Proulx <[email protected]>

Author: 3 people

analyze/analyze.go

@@ -5,15 +5,19 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/boostsecurityio/poutine/models"
+	"github.com/boostsecurityio/poutine/results"
 	"golang.org/x/sync/semaphore"
 
 	"github.com/boostsecurityio/poutine/opa"
 	"github.com/boostsecurityio/poutine/providers/pkgsupply"
-	"github.com/boostsecurityio/poutine/providers/scm/domain"
+	scm_domain "github.com/boostsecurityio/poutine/providers/scm/domain"
 	"github.com/boostsecurityio/poutine/scanner"
 	"github.com/rs/zerolog/log"
 	"github.com/schollz/progressbar/v3"
@@ -60,10 +64,13 @@ type ScmClient interface {
 
 type GitClient interface {
 	Clone(ctx context.Context, clonePath string, url string, token string, ref string) error
+	FetchCone(ctx context.Context, clonePath string, url string, token string, ref string, cone string) error
 	CommitSHA(clonePath string) (string, error)
 	LastCommitDate(ctx context.Context, clonePath string) (time.Time, error)
 	GetRemoteOriginURL(ctx context.Context, repoPath string) (string, error)
 	GetRepoHeadBranchName(ctx context.Context, repoPath string) (string, error)
+	GetUniqWorkflowsBranches(ctx context.Context, clonePath string) (map[string][]models.BranchInfo, error)
+	BlobMatches(ctx context.Context, clonePath string, blobsha string, regex *regexp.Regexp) (bool, []byte, error)
 }
 
 func NewAnalyzer(scmClient ScmClient, gitClient GitClient, formatter Formatter, config *models.Config, opaClient *opa.Opa) *Analyzer {
@@ -208,6 +215,153 @@ func (a *Analyzer) AnalyzeOrg(ctx context.Context, org string, numberOfGoroutine
 	return scannedPackages, nil
 }
 
+func (a *Analyzer) AnalyzeStaleBranches(ctx context.Context, repoString string, numberOfGoroutines *int, expand *bool, regex *regexp.Regexp) (*models.PackageInsights, error) {
+	org, repoName, err := a.ScmClient.ParseRepoAndOrg(repoString)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse repository: %w", err)
+	}
+	repo, err := a.ScmClient.GetRepo(ctx, org, repoName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get repo: %w", err)
+	}
+	provider := repo.GetProviderName()
+
+	providerVersion, err := a.ScmClient.GetProviderVersion(ctx)
+	if err != nil {
+		log.Debug().Err(err).Msgf("Failed to get provider version for %s", provider)
+	}
+
+	log.Debug().Msgf("Provider: %s, Version: %s, BaseURL: %s", provider, providerVersion, a.ScmClient.GetProviderBaseURL())
+
+	pkgsupplyClient := pkgsupply.NewStaticClient()
+
+	inventory := scanner.NewInventory(a.Opa, pkgsupplyClient, provider, providerVersion)
+
+	log.Debug().Msgf("Starting repository analysis for: %s/%s on %s", org, repoName, provider)
+	bar := a.progressBar(3, "Cloning repository")
+	_ = bar.RenderBlank()
+
+	repoUrl := repo.BuildGitURL(a.ScmClient.GetProviderBaseURL())
+	tempDir, err := a.fetchConeToTemp(ctx, repoUrl, a.ScmClient.GetToken(), "refs/heads/*:refs/remotes/origin/*", ".github/workflows")
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch cone: %w", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	bar.Describe("Listing unique workflows")
+	_ = bar.Add(1)
+
+	workflows, err := a.GitClient.GetUniqWorkflowsBranches(ctx, tempDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get unique workflow: %w", err)
+	}
+
+	bar.Describe("Check which workflows match regex: " + regex.String())
+	_ = bar.Add(1)
+
+	workflowDir := filepath.Join(tempDir, ".github", "workflows")
+	if err = os.MkdirAll(workflowDir, 0700); err != nil {
+		return nil, fmt.Errorf("failed to create .github/workflows/ dir: %w", err)
+	}
+
+	wg := sync.WaitGroup{}
+	errChan := make(chan error, 1)
+	maxGoroutines := 5
+	if numberOfGoroutines != nil {
+		maxGoroutines = *numberOfGoroutines
+	}
+	semaphore := semaphore.NewWeighted(int64(maxGoroutines))
+	m := sync.Mutex{}
+	blobShas := make([]string, 0, len(workflows))
+	for sha := range workflows {
+		blobShas = append(blobShas, sha)
+	}
+	for _, blobSha := range blobShas {
+		if err := semaphore.Acquire(ctx, 1); err != nil {
+			errChan <- fmt.Errorf("failed to acquire semaphore: %w", err)
+			break
+		}
+		wg.Add(1)
+		go func(blobSha string) {
+			defer wg.Done()
+			defer semaphore.Release(1)
+			match, content, err := a.GitClient.BlobMatches(ctx, tempDir, blobSha, regex)
+			if err != nil {
+				errChan <- fmt.Errorf("failed to blob match %s: %w", blobSha, err)
+				return
+			}
+			if match {
+				err = os.WriteFile(filepath.Join(workflowDir, blobSha+".yaml"), content, 0644)
+				if err != nil {
+					errChan <- fmt.Errorf("failed to write file for blob %s: %w", blobSha, err)
+				}
+			} else {
+				m.Lock()
+				delete(workflows, blobSha)
+				m.Unlock()
+			}
+		}(blobSha)
+	}
+	wg.Wait()
+	close(errChan)
+	for err := range errChan {
+		return nil, err
+	}
+
+	bar.Describe("Scanning package")
+	_ = bar.Add(1)
+	pkg, err := a.generatePackageInsights(ctx, tempDir, repo, "HEAD")
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate package insight: %w", err)
+	}
+
+	inventoryScanner := scanner.InventoryScanner{
+		Path: tempDir,
+		Parsers: []scanner.Parser{
+			scanner.NewGithubActionWorkflowParser(),
+		},
+	}
+
+	scannedPackage, err := inventory.ScanPackageScanner(ctx, *pkg, &inventoryScanner)
+	if err != nil {
+		return nil, fmt.Errorf("failed to scan package: %w", err)
+	}
+
+	_ = bar.Finish()
+	if *expand {
+		expanded := []results.Finding{}
+		for _, finding := range scannedPackage.FindingsResults.Findings {
+			filename := filepath.Base(finding.Meta.Path)
+			blobsha := strings.TrimSuffix(filename, filepath.Ext(filename))
+			purl, err := models.NewPurl(finding.Purl)
+			if err != nil {
+				log.Warn().Err(err).Str("purl", finding.Purl).Msg("failed to evaluate PURL, skipping")
+				continue
+			}
+			for _, branchInfo := range workflows[blobsha] {
+				for _, path := range branchInfo.FilePath {
+					finding.Meta.Path = path
+					purl.Version = branchInfo.BranchName
+					finding.Purl = purl.String()
+					expanded = append(expanded, finding)
+				}
+			}
+		}
+		scannedPackage.FindingsResults.Findings = expanded
+
+		if err := a.Formatter.Format(ctx, []*models.PackageInsights{scannedPackage}); err != nil {
+			return nil, fmt.Errorf("failed to finalize analysis of package: %w", err)
+		}
+	} else {
+		if err := a.Formatter.FormatWithPath(ctx, []*models.PackageInsights{scannedPackage}, workflows); err != nil {
+			return nil, fmt.Errorf("failed to finalize analysis of package: %w", err)
+		}
+
+	}
+
+	return scannedPackage, nil
+}
+
 func (a *Analyzer) AnalyzeRepo(ctx context.Context, repoString string, ref string) (*models.PackageInsights, error) {
 	org, repoName, err := a.ScmClient.ParseRepoAndOrg(repoString)
 	if err != nil {
@@ -304,6 +458,7 @@ func (a *Analyzer) AnalyzeLocalRepo(ctx context.Context, repoPath string) (*mode
 
 type Formatter interface {
 	Format(ctx context.Context, packages []*models.PackageInsights) error
+	FormatWithPath(ctx context.Context, packages []*models.PackageInsights, pathAssociation map[string][]models.BranchInfo) error
 }
 
 func (a *Analyzer) finalizeAnalysis(ctx context.Context, scannedPackages []*models.PackageInsights) error {
@@ -316,14 +471,15 @@ func (a *Analyzer) finalizeAnalysis(ctx context.Context, scannedPackages []*mode
 }
 
 func (a *Analyzer) generatePackageInsights(ctx context.Context, tempDir string, repo Repository, ref string) (*models.PackageInsights, error) {
+	var err error
 	commitDate, err := a.GitClient.LastCommitDate(ctx, tempDir)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get last commit date: %w", err)
+		log.Ctx(ctx).Warn().Err(err).Msg("failed to get last commit date")
 	}
 
 	commitSha, err := a.GitClient.CommitSHA(tempDir)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get commit SHA: %w", err)
+		log.Ctx(ctx).Warn().Err(err).Msg("failed to get commit SHA")
 	}
 
 	var (
@@ -376,6 +532,20 @@ func (a *Analyzer) generatePackageInsights(ctx context.Context, tempDir string,
 	return pkg, nil
 }
 
+func (a *Analyzer) fetchConeToTemp(ctx context.Context, gitURL, token, ref string, cone string) (string, error) {
+	tempDir, err := os.MkdirTemp("", TEMP_DIR_PREFIX)
+	if err != nil {
+		return "", fmt.Errorf("failed to create temp directory: %w", err)
+	}
+
+	err = a.GitClient.FetchCone(ctx, tempDir, gitURL, token, ref, cone)
+	if err != nil {
+		os.RemoveAll(tempDir) // Clean up if cloning fails
+		return "", fmt.Errorf("failed to clone repo: %w", err)
+	}
+	return tempDir, nil
+}
+
 func (a *Analyzer) cloneRepoToTemp(ctx context.Context, gitURL string, token string, ref string) (string, error) {
 	tempDir, err := os.MkdirTemp("", TEMP_DIR_PREFIX)
 	if err != nil {

cmd/analyzeRepoStaleBranches.go

@@ -0,0 +1,60 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var threadsRepoStaleBranch int
+var expand bool
+var regex string
+
+var analyzeRepoStaleBranches = &cobra.Command{
+	Use:   "analyze_repo_stale_branches",
+	Short: "Analyzes a remote repository for pull_request_target vulnerabilities in stale branches",
+	Long: `Analyzes a remote repository, looping through all remote branches to find unique GitHub Actions workflows with old pull_request_target vulnerabilities, even though the default branch does not have that vulnerability anymore.
+Example Scanning a remote Github Repository: poutine analyze_repo_stale_branches org/repo --token "$GH_TOKEN"`,
+	Args: cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		token = viper.GetString("token")
+		ctx := cmd.Context()
+		analyzer, err := GetAnalyzer(ctx, "analyze_repo_stale_branches")
+		if err != nil {
+			return fmt.Errorf("error getting analyzer analyze_repo_stale_branches: %w", err)
+		}
+
+		if Format == "sarif" {
+			return errors.New("sarif formatter not supported for analyze_repo_stale_branches")
+		}
+
+		repo := args[0]
+
+		reg, err := regexp.Compile(regex)
+		if err != nil {
+			return fmt.Errorf("error compiling regex: %w", err)
+		}
+
+		_, err = analyzer.AnalyzeStaleBranches(ctx, repo, &threadsRepoStaleBranch, &expand, reg)
+		if err != nil {
+			return fmt.Errorf("failed to analyze repo %s: %w", repo, err)
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(analyzeRepoStaleBranches)
+
+	analyzeRepoStaleBranches.Flags().StringVarP(&token, "token", "t", "", "SCM access token (env: GH_TOKEN)")
+	analyzeRepoStaleBranches.Flags().IntVarP(&threadsRepoStaleBranch, "threads", "j", 5, "Parallelization factor for scanning stale branches")
+	analyzeRepoStaleBranches.Flags().BoolVarP(&expand, "expand", "e", false, "Expand the output to the classic representation from analyze_repo")
+	analyzeRepoStaleBranches.Flags().StringVarP(&regex, "regex", "r", "pull_request_target", "Regex to check if the workflow is accessible in stale branches")
+
+	_ = viper.BindPFlag("token", analyzeRepoStaleBranches.Flags().Lookup("token"))
+	_ = viper.BindEnv("token", "GH_TOKEN")
+}