Cli on demand status

cli/docs/flags.go

@@ -153,9 +153,10 @@ const (
 	AnalyzerManagerCustomPath     = "analyzer-manager-path"
 
 	// Unique curation flags
-	CurationOutput  = "curation-format"
-	DockerImageName = "image"
-	SolutionPath    = "solution-path"
+	CurationOutput        = "curation-format"
+	DockerImageName       = "image"
+	SolutionPath          = "solution-path"
+	IncludeCachedPackages = "include-cached-packages"
 
 	// Unique git flags
 	InputFile       = "input-file"
@@ -214,7 +215,7 @@ var commandFlags = map[string][]string{
 		StaticSca, XrayLibPluginBinaryCustomPath, AnalyzerManagerCustomPath, AddSastRules,
 	},
 	CurationAudit: {
-		CurationOutput, WorkingDirs, Threads, RequirementsFile, InsecureTls, useWrapperAudit, UseIncludedBuilds, SolutionPath, DockerImageName,
+		CurationOutput, WorkingDirs, Threads, RequirementsFile, InsecureTls, useWrapperAudit, UseIncludedBuilds, SolutionPath, DockerImageName,IncludeCachedPackages,
 	},
 	GitCountContributors: {
 		InputFile, ScmType, ScmApiUrl, Token, Owner, RepoName, Months, DetailedSummary, InsecureTls,
@@ -331,6 +332,7 @@ var flagsMap = map[string]components.Flag{
 	StaticSca:                     components.NewBoolFlag(StaticSca, "Set to true to use the new SCA engine which is based on lock files.", components.SetHiddenBoolFlag()),
 	CurationOutput:                components.NewStringFlag(OutputFormat, "Defines the output format of the command. Acceptable values are: table, json.", components.WithStrDefaultValue("table")),
 	SolutionPath:                  components.NewStringFlag(SolutionPath, "Path to the .NET solution file (.sln) to use when multiple solution files are present in the directory."),
+	IncludeCachedPackages:         components.NewBoolFlag(IncludeCachedPackages, "Set to true to audit cached packages. This is also required when using Curation on-demand, as packages are cached."),
 	binarySca:                     components.NewBoolFlag(Sca, fmt.Sprintf("Selective scanners mode: Execute SCA (Software Composition Analysis) sub-scan. Use --%s to run both SCA and Contextual Analysis. Use --%s --%s to to run SCA. Can be combined with --%s.", Sca, Sca, WithoutCA, Secrets)),
 	binarySecrets:                 components.NewBoolFlag(Secrets, fmt.Sprintf("Selective scanners mode: Execute Secrets sub-scan. Can be combined with --%s.", Sca)),
 	binaryWithoutCA:               components.NewBoolFlag(WithoutCA, fmt.Sprintf("Selective scanners mode: Disable Contextual Analysis scanner after SCA. Relevant only with --%s flag.", Sca)),

cli/scancommands.go

@@ -701,6 +701,7 @@ func getCurationCommand(c *components.Context) (*curation.CurationAuditCommand,
 		SetPipRequirementsFile(c.GetStringFlagValue(flags.RequirementsFile)).
 		SetSolutionFilePath(c.GetStringFlagValue(flags.SolutionPath))
 	curationAuditCommand.SetDockerImageName(c.GetStringFlagValue(flags.DockerImageName))
+	curationAuditCommand.SetIncludeCachedPackages(c.GetBoolFlagValue(flags.IncludeCachedPackages))
 	return curationAuditCommand, nil
 }
 

commands/curation/curationaudit.go

@@ -53,12 +53,14 @@ const (
 	blocked                = "blocked"
 	BlockingReasonPolicy   = "Policy violations"
 	BlockingReasonNotFound = "Package pending update"
+	BlockingReasonOnDemand = "Package pending — Curation on-demand scan in progress"
 
 	directRelation   = "direct"
 	indirectRelation = "indirect"
 
 	BlockMessageKey  = "jfrog packages curation"
 	NotBeingFoundKey = "not being found"
+	IsOnDemand       = "on-demand"
 
 	extractPoliciesRegexTemplate = "({.*?})"
 
@@ -202,24 +204,26 @@ type PackageStatusTable struct {
 }
 
 type treeAnalyzer struct {
-	rtManager            artifactory.ArtifactoryServicesManager
-	extractPoliciesRegex *regexp.Regexp
-	rtAuth               auth.ServiceDetails
-	httpClientDetails    httputils.HttpClientDetails
-	url                  string
-	repo                 string
-	tech                 techutils.Technology
-	parallelRequests     int
-	downloadUrls         map[string]string
+	rtManager             artifactory.ArtifactoryServicesManager
+	extractPoliciesRegex  *regexp.Regexp
+	rtAuth                auth.ServiceDetails
+	httpClientDetails     httputils.HttpClientDetails
+	url                   string
+	repo                  string
+	tech                  techutils.Technology
+	parallelRequests      int
+	downloadUrls          map[string]string
+	includeCachedPackages bool
 }
 
 type CurationAuditCommand struct {
-	PackageManagerConfig *project.RepositoryConfig
-	extractPoliciesRegex *regexp.Regexp
-	workingDirs          []string
-	OriginPath           string
-	parallelRequests     int
-	dockerImageName      string
+	PackageManagerConfig  *project.RepositoryConfig
+	extractPoliciesRegex  *regexp.Regexp
+	workingDirs           []string
+	OriginPath            string
+	parallelRequests      int
+	dockerImageName       string
+	includeCachedPackages bool
 	audit.AuditParamsInterface
 }
 
@@ -265,6 +269,11 @@ func (ca *CurationAuditCommand) SetDockerImageName(dockerImageName string) *Cura
 	return ca
 }
 
+func (ca *CurationAuditCommand) SetIncludeCachedPackages(includeCachedPackages bool) *CurationAuditCommand {
+	ca.includeCachedPackages = includeCachedPackages
+	return ca
+}
+
 func (ca *CurationAuditCommand) Run() (err error) {
 	rootDir, err := os.Getwd()
 	if err != nil {
@@ -500,15 +509,16 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map
 	}
 	var packagesStatus []*PackageStatus
 	analyzer := treeAnalyzer{
-		rtManager:            rtManager,
-		extractPoliciesRegex: ca.extractPoliciesRegex,
-		rtAuth:               rtAuth,
-		httpClientDetails:    rtAuth.CreateHttpClientDetails(),
-		url:                  rtAuth.GetUrl(),
-		repo:                 ca.PackageManagerConfig.TargetRepo(),
-		tech:                 tech,
-		parallelRequests:     ca.parallelRequests,
-		downloadUrls:         depTreeResult.DownloadUrls,
+		rtManager:             rtManager,
+		extractPoliciesRegex:  ca.extractPoliciesRegex,
+		rtAuth:                rtAuth,
+		httpClientDetails:     rtAuth.CreateHttpClientDetails(),
+		url:                   rtAuth.GetUrl(),
+		repo:                  ca.PackageManagerConfig.TargetRepo(),
+		tech:                  tech,
+		parallelRequests:      ca.parallelRequests,
+		downloadUrls:          depTreeResult.DownloadUrls,
+		includeCachedPackages: ca.includeCachedPackages,
 	}
 
 	rootNodes := map[string]struct{}{}
@@ -866,7 +876,7 @@ func (nc *treeAnalyzer) fetchNodeStatus(node xrayUtils.GraphNode, p *sync.Map) e
 		if resp != nil && resp.StatusCode >= 400 && resp.StatusCode != http.StatusForbidden {
 			return errorutils.CheckErrorf(errorTemplateHeadRequest, packageUrl, name, version, resp.StatusCode, err)
 		}
-		if resp.StatusCode == http.StatusForbidden {
+		if resp.StatusCode == http.StatusForbidden || (nc.includeCachedPackages && resp.StatusCode == http.StatusOK) {
 			pkStatus, err := nc.getBlockedPackageDetails(packageUrl, name, version)
 			if err != nil {
 				return err
@@ -911,6 +921,8 @@ func (nc *treeAnalyzer) getBlockedPackageDetails(packageUrl string, name string,
 			blockingReason := BlockingReasonPolicy
 			if strings.Contains(strings.ToLower(respError.Errors[0].Message), NotBeingFoundKey) {
 				blockingReason = BlockingReasonNotFound
+			} else if strings.Contains(strings.ToLower(respError.Errors[0].Message), IsOnDemand) {
+				blockingReason = BlockingReasonOnDemand
 			}
 			policies := nc.extractPoliciesFromMsg(respError)
 			return &PackageStatus{
@@ -933,21 +945,29 @@ func (nc *treeAnalyzer) getBlockedPackageDetails(packageUrl string, name string,
 func (nc *treeAnalyzer) extractPoliciesFromMsg(respError *ErrorsResp) []Policy {
 	var policies []Policy
 	msg := respError.Errors[0].Message
-	allMatches := nc.extractPoliciesRegex.FindAllString(msg, -1)
-	for _, match := range allMatches {
-		match = strings.TrimSuffix(strings.TrimPrefix(match, "{"), "}")
-		polCond := strings.Split(match, ",")
-		if len(polCond) >= 2 {
-			pol := polCond[0]
-			cond := polCond[1]
-
-			if len(polCond) == 4 {
-				exp, rec := makeLegiblePolicyDetails(polCond[2], polCond[3])
-				policies = append(policies, Policy{Policy: strings.TrimSpace(pol),
-					Condition: strings.TrimSpace(cond), Explanation: strings.TrimSpace(exp), Recommendation: strings.TrimSpace(rec)})
-				continue
+	lowerMsg := strings.ToLower(msg)
+	switch {
+	case strings.Contains(lowerMsg, IsOnDemand):
+		policies = []Policy{{Explanation: BlockingReasonOnDemand}}
+	case strings.Contains(lowerMsg, NotBeingFoundKey):
+		policies = []Policy{{Explanation: BlockingReasonNotFound}}
+	default:
+		allMatches := nc.extractPoliciesRegex.FindAllString(msg, -1)
+		for _, match := range allMatches {
+			match = strings.TrimSuffix(strings.TrimPrefix(match, "{"), "}")
+			polCond := strings.Split(match, ",")
+			if len(polCond) >= 2 {
+				pol := polCond[0]
+				cond := polCond[1]
+
+				if len(polCond) == 4 {
+					exp, rec := makeLegiblePolicyDetails(polCond[2], polCond[3])
+					policies = append(policies, Policy{Policy: strings.TrimSpace(pol),
+						Condition: strings.TrimSpace(cond), Explanation: strings.TrimSpace(exp), Recommendation: strings.TrimSpace(rec)})
+					continue
+				}
+				policies = append(policies, Policy{Policy: strings.TrimSpace(pol), Condition: strings.TrimSpace(cond)})
 			}
-			policies = append(policies, Policy{Policy: strings.TrimSpace(pol), Condition: strings.TrimSpace(cond)})
 		}
 	}
 	return policies

commands/curation/curationaudit_test.go

@@ -131,6 +131,38 @@ func getTestCasesForExtractPoliciesFromMsg() []struct {
 			},
 			expect: nil,
 		},
+		{
+			name: "on-demand in progress",
+			errResp: &ErrorsResp{
+				Errors: []ErrorResp{
+					{
+						Status:  403,
+						Message: "Package test:1.0.0 download was blocked by JFrog Packages Curation service due to the package not being found in catalog, curation on-demand scan in progress.",
+					},
+				},
+			},
+			expect: []Policy{
+				{
+					Explanation: BlockingReasonOnDemand,
+				},
+			},
+		},
+		{
+			name: "package not found in catalog",
+			errResp: &ErrorsResp{
+				Errors: []ErrorResp{
+					{
+						Status:  403,
+						Message: "package test:1.0.0 download was blocked by jfrog packages curation service due to the package not being found in catalog",
+					},
+				},
+			},
+			expect: []Policy{
+				{
+					Explanation: BlockingReasonNotFound,
+				},
+			},
+		},
 	}
 	return tests
 }

sca/bom/buildinfo/technologies/docker/docker.go

@@ -223,6 +223,9 @@ func findDigestForPlatform(imageName, targetOS, targetArch string) (string, erro
 	buildxCmd := exec.Command("docker", "buildx", "imagetools", "inspect", imageName, "--raw")
 	buildxOutput, buildxErr := buildxCmd.CombinedOutput()
 	if buildxErr != nil {
+		if strings.Contains(string(buildxOutput), "curation service") {
+			return extractDigestFromBlockedMessage(string(buildxOutput)), nil
+		}
 		return "", fmt.Errorf("docker buildx imagetools inspect failed: %s", strings.TrimSpace(string(buildxOutput)))
 	}