diff --git a/cli/docs/flags.go b/cli/docs/flags.go index 0b97794a..4d809a5a 100644 --- a/cli/docs/flags.go +++ b/cli/docs/flags.go @@ -153,9 +153,10 @@ const ( AnalyzerManagerCustomPath = "analyzer-manager-path" // Unique curation flags - CurationOutput = "curation-format" - DockerImageName = "image" - SolutionPath = "solution-path" + CurationOutput = "curation-format" + DockerImageName = "image" + SolutionPath = "solution-path" + IncludeCachedPackages = "include-cached-packages" // Unique git flags InputFile = "input-file" @@ -214,7 +215,7 @@ var commandFlags = map[string][]string{ StaticSca, XrayLibPluginBinaryCustomPath, AnalyzerManagerCustomPath, AddSastRules, }, CurationAudit: { - CurationOutput, WorkingDirs, Threads, RequirementsFile, InsecureTls, useWrapperAudit, UseIncludedBuilds, SolutionPath, DockerImageName, + CurationOutput, WorkingDirs, Threads, RequirementsFile, InsecureTls, useWrapperAudit, UseIncludedBuilds, SolutionPath, DockerImageName,IncludeCachedPackages, }, GitCountContributors: { InputFile, ScmType, ScmApiUrl, Token, Owner, RepoName, Months, DetailedSummary, InsecureTls, @@ -331,6 +332,7 @@ var flagsMap = map[string]components.Flag{ StaticSca: components.NewBoolFlag(StaticSca, "Set to true to use the new SCA engine which is based on lock files.", components.SetHiddenBoolFlag()), CurationOutput: components.NewStringFlag(OutputFormat, "Defines the output format of the command. Acceptable values are: table, json.", components.WithStrDefaultValue("table")), SolutionPath: components.NewStringFlag(SolutionPath, "Path to the .NET solution file (.sln) to use when multiple solution files are present in the directory."), + IncludeCachedPackages: components.NewBoolFlag(IncludeCachedPackages, "Set to true to audit cached packages. This is also required when using Curation on-demand, as packages are cached."), binarySca: components.NewBoolFlag(Sca, fmt.Sprintf("Selective scanners mode: Execute SCA (Software Composition Analysis) sub-scan. Use --%s to run both SCA and Contextual Analysis. Use --%s --%s to to run SCA. Can be combined with --%s.", Sca, Sca, WithoutCA, Secrets)), binarySecrets: components.NewBoolFlag(Secrets, fmt.Sprintf("Selective scanners mode: Execute Secrets sub-scan. Can be combined with --%s.", Sca)), binaryWithoutCA: components.NewBoolFlag(WithoutCA, fmt.Sprintf("Selective scanners mode: Disable Contextual Analysis scanner after SCA. Relevant only with --%s flag.", Sca)), diff --git a/cli/scancommands.go b/cli/scancommands.go index a6ee14cd..2a79a73b 100644 --- a/cli/scancommands.go +++ b/cli/scancommands.go @@ -701,6 +701,7 @@ func getCurationCommand(c *components.Context) (*curation.CurationAuditCommand, SetPipRequirementsFile(c.GetStringFlagValue(flags.RequirementsFile)). SetSolutionFilePath(c.GetStringFlagValue(flags.SolutionPath)) curationAuditCommand.SetDockerImageName(c.GetStringFlagValue(flags.DockerImageName)) + curationAuditCommand.SetIncludeCachedPackages(c.GetBoolFlagValue(flags.IncludeCachedPackages)) return curationAuditCommand, nil } diff --git a/commands/curation/curationaudit.go b/commands/curation/curationaudit.go index 4a00738c..cac76c59 100644 --- a/commands/curation/curationaudit.go +++ b/commands/curation/curationaudit.go @@ -53,12 +53,14 @@ const ( blocked = "blocked" BlockingReasonPolicy = "Policy violations" BlockingReasonNotFound = "Package pending update" + BlockingReasonOnDemand = "Package pending — Curation on-demand scan in progress" directRelation = "direct" indirectRelation = "indirect" BlockMessageKey = "jfrog packages curation" NotBeingFoundKey = "not being found" + IsOnDemand = "on-demand" extractPoliciesRegexTemplate = "({.*?})" @@ -202,24 +204,26 @@ type PackageStatusTable struct { } type treeAnalyzer struct { - rtManager artifactory.ArtifactoryServicesManager - extractPoliciesRegex *regexp.Regexp - rtAuth auth.ServiceDetails - httpClientDetails httputils.HttpClientDetails - url string - repo string - tech techutils.Technology - parallelRequests int - downloadUrls map[string]string + rtManager artifactory.ArtifactoryServicesManager + extractPoliciesRegex *regexp.Regexp + rtAuth auth.ServiceDetails + httpClientDetails httputils.HttpClientDetails + url string + repo string + tech techutils.Technology + parallelRequests int + downloadUrls map[string]string + includeCachedPackages bool } type CurationAuditCommand struct { - PackageManagerConfig *project.RepositoryConfig - extractPoliciesRegex *regexp.Regexp - workingDirs []string - OriginPath string - parallelRequests int - dockerImageName string + PackageManagerConfig *project.RepositoryConfig + extractPoliciesRegex *regexp.Regexp + workingDirs []string + OriginPath string + parallelRequests int + dockerImageName string + includeCachedPackages bool audit.AuditParamsInterface } @@ -265,6 +269,11 @@ func (ca *CurationAuditCommand) SetDockerImageName(dockerImageName string) *Cura return ca } +func (ca *CurationAuditCommand) SetIncludeCachedPackages(includeCachedPackages bool) *CurationAuditCommand { + ca.includeCachedPackages = includeCachedPackages + return ca +} + func (ca *CurationAuditCommand) Run() (err error) { rootDir, err := os.Getwd() if err != nil { @@ -500,15 +509,16 @@ func (ca *CurationAuditCommand) auditTree(tech techutils.Technology, results map } var packagesStatus []*PackageStatus analyzer := treeAnalyzer{ - rtManager: rtManager, - extractPoliciesRegex: ca.extractPoliciesRegex, - rtAuth: rtAuth, - httpClientDetails: rtAuth.CreateHttpClientDetails(), - url: rtAuth.GetUrl(), - repo: ca.PackageManagerConfig.TargetRepo(), - tech: tech, - parallelRequests: ca.parallelRequests, - downloadUrls: depTreeResult.DownloadUrls, + rtManager: rtManager, + extractPoliciesRegex: ca.extractPoliciesRegex, + rtAuth: rtAuth, + httpClientDetails: rtAuth.CreateHttpClientDetails(), + url: rtAuth.GetUrl(), + repo: ca.PackageManagerConfig.TargetRepo(), + tech: tech, + parallelRequests: ca.parallelRequests, + downloadUrls: depTreeResult.DownloadUrls, + includeCachedPackages: ca.includeCachedPackages, } rootNodes := map[string]struct{}{} @@ -866,7 +876,7 @@ func (nc *treeAnalyzer) fetchNodeStatus(node xrayUtils.GraphNode, p *sync.Map) e if resp != nil && resp.StatusCode >= 400 && resp.StatusCode != http.StatusForbidden { return errorutils.CheckErrorf(errorTemplateHeadRequest, packageUrl, name, version, resp.StatusCode, err) } - if resp.StatusCode == http.StatusForbidden { + if resp.StatusCode == http.StatusForbidden || (nc.includeCachedPackages && resp.StatusCode == http.StatusOK) { pkStatus, err := nc.getBlockedPackageDetails(packageUrl, name, version) if err != nil { return err @@ -911,6 +921,8 @@ func (nc *treeAnalyzer) getBlockedPackageDetails(packageUrl string, name string, blockingReason := BlockingReasonPolicy if strings.Contains(strings.ToLower(respError.Errors[0].Message), NotBeingFoundKey) { blockingReason = BlockingReasonNotFound + } else if strings.Contains(strings.ToLower(respError.Errors[0].Message), IsOnDemand) { + blockingReason = BlockingReasonOnDemand } policies := nc.extractPoliciesFromMsg(respError) return &PackageStatus{ @@ -933,21 +945,29 @@ func (nc *treeAnalyzer) getBlockedPackageDetails(packageUrl string, name string, func (nc *treeAnalyzer) extractPoliciesFromMsg(respError *ErrorsResp) []Policy { var policies []Policy msg := respError.Errors[0].Message - allMatches := nc.extractPoliciesRegex.FindAllString(msg, -1) - for _, match := range allMatches { - match = strings.TrimSuffix(strings.TrimPrefix(match, "{"), "}") - polCond := strings.Split(match, ",") - if len(polCond) >= 2 { - pol := polCond[0] - cond := polCond[1] - - if len(polCond) == 4 { - exp, rec := makeLegiblePolicyDetails(polCond[2], polCond[3]) - policies = append(policies, Policy{Policy: strings.TrimSpace(pol), - Condition: strings.TrimSpace(cond), Explanation: strings.TrimSpace(exp), Recommendation: strings.TrimSpace(rec)}) - continue + lowerMsg := strings.ToLower(msg) + switch { + case strings.Contains(lowerMsg, IsOnDemand): + policies = []Policy{{Explanation: BlockingReasonOnDemand}} + case strings.Contains(lowerMsg, NotBeingFoundKey): + policies = []Policy{{Explanation: BlockingReasonNotFound}} + default: + allMatches := nc.extractPoliciesRegex.FindAllString(msg, -1) + for _, match := range allMatches { + match = strings.TrimSuffix(strings.TrimPrefix(match, "{"), "}") + polCond := strings.Split(match, ",") + if len(polCond) >= 2 { + pol := polCond[0] + cond := polCond[1] + + if len(polCond) == 4 { + exp, rec := makeLegiblePolicyDetails(polCond[2], polCond[3]) + policies = append(policies, Policy{Policy: strings.TrimSpace(pol), + Condition: strings.TrimSpace(cond), Explanation: strings.TrimSpace(exp), Recommendation: strings.TrimSpace(rec)}) + continue + } + policies = append(policies, Policy{Policy: strings.TrimSpace(pol), Condition: strings.TrimSpace(cond)}) } - policies = append(policies, Policy{Policy: strings.TrimSpace(pol), Condition: strings.TrimSpace(cond)}) } } return policies diff --git a/commands/curation/curationaudit_test.go b/commands/curation/curationaudit_test.go index c4617e35..2053f949 100644 --- a/commands/curation/curationaudit_test.go +++ b/commands/curation/curationaudit_test.go @@ -131,6 +131,38 @@ func getTestCasesForExtractPoliciesFromMsg() []struct { }, expect: nil, }, + { + name: "on-demand in progress", + errResp: &ErrorsResp{ + Errors: []ErrorResp{ + { + Status: 403, + Message: "Package test:1.0.0 download was blocked by JFrog Packages Curation service due to the package not being found in catalog, curation on-demand scan in progress.", + }, + }, + }, + expect: []Policy{ + { + Explanation: BlockingReasonOnDemand, + }, + }, + }, + { + name: "package not found in catalog", + errResp: &ErrorsResp{ + Errors: []ErrorResp{ + { + Status: 403, + Message: "package test:1.0.0 download was blocked by jfrog packages curation service due to the package not being found in catalog", + }, + }, + }, + expect: []Policy{ + { + Explanation: BlockingReasonNotFound, + }, + }, + }, } return tests } diff --git a/sca/bom/buildinfo/technologies/docker/docker.go b/sca/bom/buildinfo/technologies/docker/docker.go index 181c9512..b156d7fa 100644 --- a/sca/bom/buildinfo/technologies/docker/docker.go +++ b/sca/bom/buildinfo/technologies/docker/docker.go @@ -223,6 +223,9 @@ func findDigestForPlatform(imageName, targetOS, targetArch string) (string, erro buildxCmd := exec.Command("docker", "buildx", "imagetools", "inspect", imageName, "--raw") buildxOutput, buildxErr := buildxCmd.CombinedOutput() if buildxErr != nil { + if strings.Contains(string(buildxOutput), "curation service") { + return extractDigestFromBlockedMessage(string(buildxOutput)), nil + } return "", fmt.Errorf("docker buildx imagetools inspect failed: %s", strings.TrimSpace(string(buildxOutput))) }