Skip to content

Commit

Permalink
Update RustCli Parsing to process pkgId, and introduce manual override (
Browse files Browse the repository at this point in the history 
#1106)

* Update RustCli Parsing to process pkgId, and allow manual override to fallback with DisableRustCliScan

* add tests

* Update detector version

* Update cli detector to use manifest packages instead of manually parsing
  • Loading branch information
@FernandoRojo
FernandoRojo authored May 16, 2024
1 parent 04776cc commit 5894c27
Show file tree
Hide file tree
Showing 3 changed files with 139 additions and 40 deletions.
6 changes: 5 additions & 1 deletion src/Microsoft.ComponentDetection.Contracts/TypedComponent/CargoComponent.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,13 @@ private CargoComponent()
// reserved for deserialization
}

public CargoComponent(string name, string version, string author = null, string license = null)
public CargoComponent(string name, string version, string author = null, string license = null, string source = null)
{
this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Cargo));
this.Version = this.ValidateRequiredInput(version, nameof(this.Version), nameof(ComponentType.Cargo));
this.Author = author;
this.License = license;
this.Source = source;
}

public string Name { get; set; }
Expand All @@ -28,6 +29,9 @@ public CargoComponent(string name, string version, string author = null, string

[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public string? License { get; set; }

[JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
public string? Source { get; set; }
#nullable disable

public override ComponentType Type => ComponentType.Cargo;
Expand Down
57 changes: 37 additions & 20 deletions src/Microsoft.ComponentDetection.Detectors/rust/RustCliDetector.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ namespace Microsoft.ComponentDetection.Detectors.Rust;
using Microsoft.ComponentDetection.Contracts.TypedComponent;
using Microsoft.ComponentDetection.Detectors.Rust.Contracts;
using Microsoft.Extensions.Logging;
using MoreLinq.Extensions;
using Newtonsoft.Json;
using Tomlyn;

Expand All @@ -21,8 +22,7 @@ namespace Microsoft.ComponentDetection.Detectors.Rust;
/// </summary>
public class RustCliDetector : FileComponentDetector
{
//// PkgName[ Version][ (Source)]
private static readonly Regex DependencyFormatRegex = new Regex(
private static readonly Regex DependencyFormatRegexCargoLock = new Regex(
@"^(?<packageName>[^ ]+)(?: (?<version>[^ ]+))?(?: \((?<source>[^()]*)\))?$",
RegexOptions.Compiled);

Expand All @@ -33,22 +33,27 @@ public class RustCliDetector : FileComponentDetector

private readonly ICommandLineInvocationService cliService;

private readonly IEnvironmentVariableService envVarService;

/// <summary>
/// Initializes a new instance of the <see cref="RustCliDetector"/> class.
/// </summary>
/// <param name="componentStreamEnumerableFactory">The component stream enumerable factory.</param>
/// <param name="walkerFactory">The walker factory.</param>
/// <param name="cliService">The command line invocation service.</param>
/// <param name="envVarService">The environment variable reader service.</param>
/// <param name="logger">The logger.</param>
public RustCliDetector(
IComponentStreamEnumerableFactory componentStreamEnumerableFactory,
IObservableDirectoryWalkerFactory walkerFactory,
ICommandLineInvocationService cliService,
IEnvironmentVariableService envVarService,
ILogger<RustCliDetector> logger)
{
this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory;
this.Scanner = walkerFactory;
this.cliService = cliService;
this.envVarService = envVarService;
this.Logger = logger;
}

Expand All @@ -62,7 +67,7 @@ public RustCliDetector(
public override IEnumerable<ComponentType> SupportedComponentTypes => new[] { ComponentType.Cargo };

/// <inheritdoc />
public override int Version => 3;
public override int Version => 4;

/// <inheritdoc />
public override IList<string> SearchPatterns { get; } = new[] { "Cargo.toml" };
Expand All @@ -77,7 +82,14 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID

try
{
if (!await this.cliService.CanCommandBeLocatedAsync("cargo", null))
if (this.IsRustCliManuallyDisabled())
{
this.Logger.LogWarning("Rust Cli has been manually disabled, fallback strategy performed.");
record.DidRustCliCommandFail = false;
record.WasRustFallbackStrategyUsed = true;
record.FallbackReason = "Manually Disabled";
}
else if (!await this.cliService.CanCommandBeLocatedAsync("cargo", null))
{
this.Logger.LogWarning("Could not locate cargo command. Skipping Rust CLI detection");
record.DidRustCliCommandFail = true;
Expand Down Expand Up @@ -112,10 +124,13 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
var graph = BuildGraph(metadata);

var packages = metadata.Packages.ToDictionary(
x => $"{x.Name} {x.Version}",
x => (
x => $"{x.Id}",
x => new CargoComponent(
x.Name,
x.Version,
(x.Authors == null || x.Authors.Any(a => string.IsNullOrWhiteSpace(a)) || !x.Authors.Any()) ? null : string.Join(", ", x.Authors),
string.IsNullOrWhiteSpace(x.License) ? null : x.License));
string.IsNullOrWhiteSpace(x.License) ? null : x.License,
x.Source));

var root = metadata.Resolve.Root;
HashSet<string> visitedDependencies = new();
Expand Down Expand Up @@ -187,9 +202,9 @@ private static bool ShouldFallbackFromError(string error)
return true;
}

private static bool ParseDependency(string dependency, out string packageName, out string version, out string source)
private static bool ParseDependencyCargoLock(string dependency, out string packageName, out string version, out string source)
{
var match = DependencyFormatRegex.Match(dependency);
var match = DependencyFormatRegexCargoLock.Match(dependency);
var packageNameMatch = match.Groups["packageName"];
var versionMatch = match.Groups["version"];
var sourceMatch = match.Groups["source"];
Expand All @@ -206,41 +221,43 @@ private static bool ParseDependency(string dependency, out string packageName, o
return match.Success;
}

private bool IsRustCliManuallyDisabled()
{
return this.envVarService.IsEnvironmentVariableValueTrue("DisableRustCliScan");
}

private void TraverseAndRecordComponents(
ISingleFileComponentRecorder recorder,
string location,
IReadOnlyDictionary<string, Node> graph,
string id,
DetectedComponent parent,
Dep depInfo,
IReadOnlyDictionary<string, (string Authors, string License)> packagesMetadata,
IReadOnlyDictionary<string, CargoComponent> packagesMetadata,
ISet<string> visitedDependencies,
bool explicitlyReferencedDependency = false,
bool isTomlRoot = false)
{
try
{
var isDevelopmentDependency = depInfo?.DepKinds.Any(x => x.Kind is Kind.Dev) ?? false;
if (!ParseDependency(id, out var name, out var version, out var source))

if (!packagesMetadata.TryGetValue($"{id}", out var cargoComponent))
{
// Could not parse the dependency string
this.Logger.LogWarning("Failed to parse dependency '{Id}'", id);
this.Logger.LogWarning("Did not find dependency '{Id}' in Manifest.packages, skipping", id);
return;
}

var (authors, license) = packagesMetadata.TryGetValue($"{name} {version}", out var package)
? package
: (null, null);

var detectedComponent = new DetectedComponent(new CargoComponent(name, version, authors, license));
var detectedComponent = new DetectedComponent(cargoComponent);

if (!graph.TryGetValue(id, out var node))
{
this.Logger.LogWarning("Could not find {Id} at {Location} in cargo metadata output", id, location);
return;
}

var shouldRegister = !isTomlRoot && !source.StartsWith("path+file");
var shouldRegister = !isTomlRoot && cargoComponent.Source != null;
if (shouldRegister)
{
recorder.RegisterUsage(
Expand Down Expand Up @@ -298,7 +315,7 @@ private async Task ProcessCargoLockFallbackAsync(IComponentStream cargoTomlFile,
var cargoLockFileStream = this.FindCorrespondingCargoLock(cargoTomlFile, singleFileComponentRecorder);
if (cargoLockFileStream == null)
{
this.Logger.LogWarning("Could not find Cargo.lock file for {CargoTomlLocation}, skipping processing", cargoTomlFile.Location);
this.Logger.LogWarning("Fallback failed, could not find Cargo.lock file for {CargoTomlLocation}, skipping processing", cargoTomlFile.Location);
record.FallbackCargoLockFound = false;
return;
}
Expand Down Expand Up @@ -402,7 +419,7 @@ private void ProcessDependency(
try
{
// Extract the information from the dependency (name with optional version and source)
if (!ParseDependency(dependency, out var childName, out var childVersion, out var childSource))
if (!ParseDependencyCargoLock(dependency, out var childName, out var childVersion, out var childSource))
{
// Could not parse the dependency string
throw new FormatException($"Failed to parse dependency '{dependency}'");
Expand Down
Loading

0 comments on commit 5894c27

Please sign in to comment.